nnetsauce

 1from .base.base import Base
 2from .base.baseRegressor import BaseRegressor
 3from .boosting.adaBoostClassifier import AdaBoostClassifier
 4from .custom.customClassifier import CustomClassifier
 5from .custom.customRegressor import CustomRegressor
 6from .datasets import Downloader
 7from .deep.deepClassifier import DeepClassifier
 8from .deep.deepRegressor import DeepRegressor
 9from .deep.deepMTS import DeepMTS
10from .glm.glmClassifier import GLMClassifier
11from .glm.glmRegressor import GLMRegressor
12from .lazypredict.lazyClassifier import LazyClassifier
13from .lazypredict.lazyRegressor import LazyRegressor
14from .lazypredict.lazydeepClassifier import LazyDeepClassifier
15from .lazypredict.lazydeepRegressor import LazyDeepRegressor
16from .lazypredict.lazydeepMTS import LazyDeepMTS
17from .mts.mts import MTS
18from .mts.classical import ClassicalMTS
19from .multitask.multitaskClassifier import MultitaskClassifier
20from .multitask.simplemultitaskClassifier import SimpleMultitaskClassifier
21from .optimizers.optimizer import Optimizer
22from .predictioninterval import PredictionInterval
23from .randombag.randomBagClassifier import RandomBagClassifier
24from .randombag.randomBagRegressor import RandomBagRegressor
25from .ridge2.ridge2Classifier import Ridge2Classifier
26from .ridge2.ridge2Regressor import Ridge2Regressor
27from .ridge2.ridge2MultitaskClassifier import Ridge2MultitaskClassifier
28from .rvfl.bayesianrvflRegressor import BayesianRVFLRegressor
29from .rvfl.bayesianrvfl2Regressor import BayesianRVFL2Regressor
30from .sampling import SubSampler
31
32__all__ = [
33    "AdaBoostClassifier",
34    "Base",
35    "BaseRegressor",
36    "BayesianRVFLRegressor",
37    "BayesianRVFL2Regressor",
38    "ClassicalMTS",
39    "CustomClassifier",
40    "CustomRegressor",
41    "DeepClassifier",
42    "DeepRegressor",
43    "DeepMTS",
44    "Downloader",
45    "GLMClassifier",
46    "GLMRegressor",
47    "LazyClassifier",
48    "LazyRegressor",
49    "LazyDeepClassifier",
50    "LazyDeepRegressor",
51    "LazyDeepMTS",
52    "MTS",
53    "MultitaskClassifier",
54    "PredictionInterval",
55    "SimpleMultitaskClassifier",
56    "Optimizer",
57    "RandomBagRegressor",
58    "RandomBagClassifier",
59    "Ridge2Regressor",
60    "Ridge2Classifier",
61    "Ridge2MultitaskClassifier",
62    "SubSampler",
63]
class AdaBoostClassifier(nnetsauce.boosting.bst.Boosting, sklearn.base.ClassifierMixin):
 21class AdaBoostClassifier(Boosting, ClassifierMixin):
 22    """AdaBoost Classification (SAMME) model class derived from class Boosting
 23
 24    Parameters:
 25
 26        obj: object
 27            any object containing a method fit (obj.fit()) and a method predict
 28            (obj.predict())
 29
 30        n_estimators: int
 31            number of boosting iterations
 32
 33        learning_rate: float
 34            learning rate of the boosting procedure
 35
 36        n_hidden_features: int
 37            number of nodes in the hidden layer
 38
 39        reg_lambda: float
 40            regularization parameter for weights
 41
 42        reg_alpha: float
 43            controls compromize between l1 and l2 norm of weights
 44
 45        activation_name: str
 46            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 47
 48        a: float
 49            hyperparameter for 'prelu' or 'elu' activation function
 50
 51        nodes_sim: str
 52            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 53            'uniform'
 54
 55        bias: boolean
 56            indicates if the hidden layer contains a bias term (True) or not
 57            (False)
 58
 59        dropout: float
 60            regularization parameter; (random) percentage of nodes dropped out
 61            of the training
 62
 63        direct_link: boolean
 64            indicates if the original predictors are included (True) in model's
 65            fitting or not (False)
 66
 67        n_clusters: int
 68            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 69                no clustering)
 70
 71        cluster_encode: bool
 72            defines how the variable containing clusters is treated (default is one-hot)
 73            if `False`, then labels are used, without one-hot encoding
 74
 75        type_clust: str
 76            type of clustering method: currently k-means ('kmeans') or Gaussian
 77            Mixture Model ('gmm')
 78
 79        type_scaling: a tuple of 3 strings
 80            scaling methods for inputs, hidden layer, and clustering respectively
 81            (and when relevant).
 82            Currently available: standardization ('std') or MinMax scaling ('minmax')
 83
 84        col_sample: float
 85            percentage of covariates randomly chosen for training
 86
 87        row_sample: float
 88            percentage of rows chosen for training, by stratified bootstrapping
 89
 90        seed: int
 91            reproducibility seed for nodes_sim=='uniform'
 92
 93        verbose: int
 94            0 for no output, 1 for a progress bar (default is 1)
 95
 96        method: str
 97            type of Adaboost method, 'SAMME' (discrete) or 'SAMME.R' (real)
 98
 99        backend: str
100            "cpu" or "gpu" or "tpu"
101
102    Attributes:
103
104        alpha_: list
105            AdaBoost coefficients alpha_m
106
107        base_learners_: dict
108            a dictionary containing the base learners
109
110    Examples:
111
112    See also [https://github.com/Techtonique/nnetsauce/blob/master/examples/adaboost_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/adaboost_classification.py)
113
114    ```python
115    import nnetsauce as ns
116    import numpy as np
117    from sklearn.datasets import load_breast_cancer
118    from sklearn.linear_model import LogisticRegression
119    from sklearn.model_selection import train_test_split
120    from sklearn import metrics
121    from time import time
122
123    breast_cancer = load_breast_cancer()
124    Z = breast_cancer.data
125    t = breast_cancer.target
126    np.random.seed(123)
127    X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2)
128
129    # SAMME.R
130    clf = LogisticRegression(solver='liblinear', multi_class = 'ovr',
131                            random_state=123)
132    fit_obj = ns.AdaBoostClassifier(clf,
133                                    n_hidden_features=int(11.22338867),
134                                    direct_link=True,
135                                    n_estimators=250, learning_rate=0.01126343,
136                                    col_sample=0.72684326, row_sample=0.86429443,
137                                    dropout=0.63078613, n_clusters=2,
138                                    type_clust="gmm",
139                                    verbose=1, seed = 123,
140                                    method="SAMME.R")
141
142    start = time()
143    fit_obj.fit(X_train, y_train)
144    print(f"Elapsed {time() - start}")
145
146    start = time()
147    print(fit_obj.score(X_test, y_test))
148    print(f"Elapsed {time() - start}")
149
150    preds = fit_obj.predict(X_test)
151
152    print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
153    print(metrics.classification_report(preds, y_test))
154
155    ```
156
157    """
158
159    # construct the object -----
160
161    def __init__(
162        self,
163        obj,
164        n_estimators=10,
165        learning_rate=0.1,
166        n_hidden_features=1,
167        reg_lambda=0,
168        reg_alpha=0.5,
169        activation_name="relu",
170        a=0.01,
171        nodes_sim="sobol",
172        bias=True,
173        dropout=0,
174        direct_link=False,
175        n_clusters=2,
176        cluster_encode=True,
177        type_clust="kmeans",
178        type_scaling=("std", "std", "std"),
179        col_sample=1,
180        row_sample=1,
181        seed=123,
182        verbose=1,
183        method="SAMME",
184        backend="cpu",
185    ):
186        self.type_fit = "classification"
187        self.verbose = verbose
188        self.method = method
189        self.reg_lambda = reg_lambda
190        self.reg_alpha = reg_alpha
191
192        super().__init__(
193            obj=obj,
194            n_estimators=n_estimators,
195            learning_rate=learning_rate,
196            n_hidden_features=n_hidden_features,
197            activation_name=activation_name,
198            a=a,
199            nodes_sim=nodes_sim,
200            bias=bias,
201            dropout=dropout,
202            direct_link=direct_link,
203            n_clusters=n_clusters,
204            cluster_encode=cluster_encode,
205            type_clust=type_clust,
206            type_scaling=type_scaling,
207            col_sample=col_sample,
208            row_sample=row_sample,
209            seed=seed,
210            backend=backend,
211        )
212
213        self.alpha_ = []
214        self.base_learners_ = dict.fromkeys(range(n_estimators))
215
216    def fit(self, X, y, sample_weight=None, **kwargs):
217        """Fit Boosting model to training data (X, y).
218
219        Parameters:
220
221            X: {array-like}, shape = [n_samples, n_features]
222                Training vectors, where n_samples is the number
223                of samples and n_features is the number of features.
224
225            y: array-like, shape = [n_samples]
226                Target values.
227
228            **kwargs: additional parameters to be passed to
229                    self.cook_training_set or self.obj.fit
230
231        Returns:
232
233             self: object
234        """
235
236        assert mx.is_factor(y), "y must contain only integers"
237
238        assert self.method in (
239            "SAMME",
240            "SAMME.R",
241        ), "`method` must be either 'SAMME' or 'SAMME.R'"
242
243        assert (self.reg_lambda <= 1) & (
244            self.reg_lambda >= 0
245        ), "must have self.reg_lambda <= 1 &  self.reg_lambda >= 0"
246
247        assert (self.reg_alpha <= 1) & (
248            self.reg_alpha >= 0
249        ), "must have self.reg_alpha <= 1 &  self.reg_alpha >= 0"
250
251        # training
252        n, p = X.shape
253        self.n_classes = len(np.unique(y))
254        self.classes_ = np.unique(y)  # for compatibility with sklearn
255        self.n_classes_ = len(self.classes_)  # for compatibility with sklearn
256
257        if sample_weight is None:
258            w_m = np.repeat(1.0 / n, n)
259        else:
260            w_m = np.asarray(sample_weight)
261
262        base_learner = CustomClassifier(
263            self.obj,
264            n_hidden_features=self.n_hidden_features,
265            activation_name=self.activation_name,
266            a=self.a,
267            nodes_sim=self.nodes_sim,
268            bias=self.bias,
269            dropout=self.dropout,
270            direct_link=self.direct_link,
271            n_clusters=self.n_clusters,
272            type_clust=self.type_clust,
273            type_scaling=self.type_scaling,
274            col_sample=self.col_sample,
275            row_sample=self.row_sample,
276            seed=self.seed,
277        )
278
279        if self.verbose == 1:
280            pbar = Progbar(self.n_estimators)
281
282        if self.method == "SAMME":
283            err_m = 1e6
284            err_bound = 1 - 1 / self.n_classes
285            self.alpha_.append(1.0)
286            x_range_n = range(n)
287
288            for m in range(self.n_estimators):
289                preds = base_learner.fit(
290                    X, y, sample_weight=np.ravel(w_m, order="C"), **kwargs
291                ).predict(X)
292
293                self.base_learners_.update(
294                    {m: pickle.loads(pickle.dumps(base_learner, -1))}
295                )
296
297                cond = [y[i] != preds[i] for i in x_range_n]
298
299                err_m = max(
300                    sum([elt[0] * elt[1] for elt in zip(cond, w_m)]),
301                    2.220446049250313e-16,
302                )  # sum(w_m) == 1
303
304                if self.reg_lambda > 0:
305                    err_m += self.reg_lambda * (
306                        (1 - self.reg_alpha) * 0.5 * sum([x**2 for x in w_m])
307                        + self.reg_alpha * sum([abs(x) for x in w_m])
308                    )
309
310                err_m = min(err_m, err_bound)
311
312                alpha_m = self.learning_rate * log(
313                    (self.n_classes - 1) * (1 - err_m) / err_m
314                )
315
316                self.alpha_.append(alpha_m)
317
318                w_m_temp = [exp(alpha_m * cond[i]) for i in x_range_n]
319
320                sum_w_m = sum(w_m_temp)
321
322                w_m = np.asarray([w_m_temp[i] / sum_w_m for i in x_range_n])
323
324                base_learner.set_params(seed=self.seed + (m + 1) * 1000)
325
326                if self.verbose == 1:
327                    pbar.update(m)
328
329            if self.verbose == 1:
330                pbar.update(self.n_estimators)
331
332            self.n_estimators = len(self.base_learners_)
333            self.classes_ = np.unique(y)
334
335            return self
336
337        if self.method == "SAMME.R":
338            Y = mo.one_hot_encode2(y, self.n_classes)
339
340            if sample_weight is None:
341                w_m = np.repeat(1.0 / n, n)  # (N, 1)
342
343            else:
344                w_m = np.asarray(sample_weight)
345
346            for m in range(self.n_estimators):
347                probs = base_learner.fit(
348                    X, y, sample_weight=np.ravel(w_m, order="C"), **kwargs
349                ).predict_proba(X)
350
351                np.clip(
352                    a=probs, a_min=2.220446049250313e-16, a_max=1.0, out=probs
353                )
354
355                self.base_learners_.update(
356                    {m: pickle.loads(pickle.dumps(base_learner, -1))}
357                )
358
359                w_m *= np.exp(
360                    -1.0
361                    * self.learning_rate
362                    * (1.0 - 1.0 / self.n_classes)
363                    * xlogy(Y, probs).sum(axis=1)
364                )
365
366                w_m /= np.sum(w_m)
367
368                base_learner.set_params(seed=self.seed + (m + 1) * 1000)
369
370                if self.verbose == 1:
371                    pbar.update(m)
372
373            if self.verbose == 1:
374                pbar.update(self.n_estimators)
375
376            self.n_estimators = len(self.base_learners_)
377            self.classes_ = np.unique(y)
378
379            return self
380
381    def predict(self, X, **kwargs):
382        """Predict test data X.
383
384        Parameters:
385
386            X: {array-like}, shape = [n_samples, n_features]
387                Training vectors, where n_samples is the number
388                of samples and n_features is the number of features.
389
390            **kwargs: additional parameters to be passed to
391                  self.cook_test_set
392
393        Returns:
394
395            model predictions: {array-like}
396        """
397        return self.predict_proba(X, **kwargs).argmax(axis=1)
398
399    def predict_proba(self, X, **kwargs):
400        """Predict probabilities for test data X.
401
402        Parameters:
403
404            X: {array-like}, shape = [n_samples, n_features]
405                Training vectors, where n_samples is the number
406                of samples and n_features is the number of features.
407
408            **kwargs: additional parameters to be passed to
409                  self.cook_test_set
410
411        Returns:
412
413            probability estimates for test data: {array-like}
414
415        """
416
417        n_iter = len(self.base_learners_)
418
419        if self.method == "SAMME":
420            ensemble_learner = np.zeros((X.shape[0], self.n_classes))
421
422            # if self.verbose == 1:
423            #    pbar = Progbar(n_iter)
424
425            for idx, base_learner in self.base_learners_.items():
426                preds = base_learner.predict(X, **kwargs)
427
428                ensemble_learner += self.alpha_[idx] * mo.one_hot_encode2(
429                    preds, self.n_classes
430                )
431
432                # if self.verbose == 1:
433                #    pbar.update(idx)
434
435            # if self.verbose == 1:
436            #    pbar.update(n_iter)
437
438            expit_ensemble_learner = expit(ensemble_learner)
439
440            sum_ensemble = expit_ensemble_learner.sum(axis=1)
441
442            return expit_ensemble_learner / sum_ensemble[:, None]
443
444        # if self.method == "SAMME.R":
445        ensemble_learner = 0
446
447        # if self.verbose == 1:
448        #    pbar = Progbar(n_iter)
449
450        for idx, base_learner in self.base_learners_.items():
451            probs = base_learner.predict_proba(X, **kwargs)
452
453            np.clip(a=probs, a_min=2.220446049250313e-16, a_max=1.0, out=probs)
454
455            log_preds_proba = np.log(probs)
456
457            ensemble_learner += (
458                log_preds_proba - log_preds_proba.mean(axis=1)[:, None]
459            )
460
461            # if self.verbose == 1:
462            #    pbar.update(idx)
463
464        ensemble_learner *= self.n_classes - 1
465
466        # if self.verbose == 1:
467        #    pbar.update(n_iter)
468
469        expit_ensemble_learner = expit(ensemble_learner)
470
471        sum_ensemble = expit_ensemble_learner.sum(axis=1)
472
473        return expit_ensemble_learner / sum_ensemble[:, None]

AdaBoost Classification (SAMME) model class derived from class Boosting

Parameters:

obj: object
    any object containing a method fit (obj.fit()) and a method predict
    (obj.predict())

n_estimators: int
    number of boosting iterations

learning_rate: float
    learning rate of the boosting procedure

n_hidden_features: int
    number of nodes in the hidden layer

reg_lambda: float
    regularization parameter for weights

reg_alpha: float
    controls compromize between l1 and l2 norm of weights

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original predictors are included (True) in model's
    fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

col_sample: float
    percentage of covariates randomly chosen for training

row_sample: float
    percentage of rows chosen for training, by stratified bootstrapping

seed: int
    reproducibility seed for nodes_sim=='uniform'

verbose: int
    0 for no output, 1 for a progress bar (default is 1)

method: str
    type of Adaboost method, 'SAMME' (discrete) or 'SAMME.R' (real)

backend: str
    "cpu" or "gpu" or "tpu"

Attributes:

alpha_: list
    AdaBoost coefficients alpha_m

base_learners_: dict
    a dictionary containing the base learners

Examples:

See also https://github.com/Techtonique/nnetsauce/blob/master/examples/adaboost_classification.py

import nnetsauce as ns
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn import metrics
from time import time

breast_cancer = load_breast_cancer()
Z = breast_cancer.data
t = breast_cancer.target
np.random.seed(123)
X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2)

# SAMME.R
clf = LogisticRegression(solver='liblinear', multi_class = 'ovr',
                        random_state=123)
fit_obj = ns.AdaBoostClassifier(clf,
                                n_hidden_features=int(11.22338867),
                                direct_link=True,
                                n_estimators=250, learning_rate=0.01126343,
                                col_sample=0.72684326, row_sample=0.86429443,
                                dropout=0.63078613, n_clusters=2,
                                type_clust="gmm",
                                verbose=1, seed = 123,
                                method="SAMME.R")

start = time()
fit_obj.fit(X_train, y_train)
print(f"Elapsed {time() - start}")

start = time()
print(fit_obj.score(X_test, y_test))
print(f"Elapsed {time() - start}")

preds = fit_obj.predict(X_test)

print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
print(metrics.classification_report(preds, y_test))
def fit(self, X, y, sample_weight=None, **kwargs):
216    def fit(self, X, y, sample_weight=None, **kwargs):
217        """Fit Boosting model to training data (X, y).
218
219        Parameters:
220
221            X: {array-like}, shape = [n_samples, n_features]
222                Training vectors, where n_samples is the number
223                of samples and n_features is the number of features.
224
225            y: array-like, shape = [n_samples]
226                Target values.
227
228            **kwargs: additional parameters to be passed to
229                    self.cook_training_set or self.obj.fit
230
231        Returns:
232
233             self: object
234        """
235
236        assert mx.is_factor(y), "y must contain only integers"
237
238        assert self.method in (
239            "SAMME",
240            "SAMME.R",
241        ), "`method` must be either 'SAMME' or 'SAMME.R'"
242
243        assert (self.reg_lambda <= 1) & (
244            self.reg_lambda >= 0
245        ), "must have self.reg_lambda <= 1 &  self.reg_lambda >= 0"
246
247        assert (self.reg_alpha <= 1) & (
248            self.reg_alpha >= 0
249        ), "must have self.reg_alpha <= 1 &  self.reg_alpha >= 0"
250
251        # training
252        n, p = X.shape
253        self.n_classes = len(np.unique(y))
254        self.classes_ = np.unique(y)  # for compatibility with sklearn
255        self.n_classes_ = len(self.classes_)  # for compatibility with sklearn
256
257        if sample_weight is None:
258            w_m = np.repeat(1.0 / n, n)
259        else:
260            w_m = np.asarray(sample_weight)
261
262        base_learner = CustomClassifier(
263            self.obj,
264            n_hidden_features=self.n_hidden_features,
265            activation_name=self.activation_name,
266            a=self.a,
267            nodes_sim=self.nodes_sim,
268            bias=self.bias,
269            dropout=self.dropout,
270            direct_link=self.direct_link,
271            n_clusters=self.n_clusters,
272            type_clust=self.type_clust,
273            type_scaling=self.type_scaling,
274            col_sample=self.col_sample,
275            row_sample=self.row_sample,
276            seed=self.seed,
277        )
278
279        if self.verbose == 1:
280            pbar = Progbar(self.n_estimators)
281
282        if self.method == "SAMME":
283            err_m = 1e6
284            err_bound = 1 - 1 / self.n_classes
285            self.alpha_.append(1.0)
286            x_range_n = range(n)
287
288            for m in range(self.n_estimators):
289                preds = base_learner.fit(
290                    X, y, sample_weight=np.ravel(w_m, order="C"), **kwargs
291                ).predict(X)
292
293                self.base_learners_.update(
294                    {m: pickle.loads(pickle.dumps(base_learner, -1))}
295                )
296
297                cond = [y[i] != preds[i] for i in x_range_n]
298
299                err_m = max(
300                    sum([elt[0] * elt[1] for elt in zip(cond, w_m)]),
301                    2.220446049250313e-16,
302                )  # sum(w_m) == 1
303
304                if self.reg_lambda > 0:
305                    err_m += self.reg_lambda * (
306                        (1 - self.reg_alpha) * 0.5 * sum([x**2 for x in w_m])
307                        + self.reg_alpha * sum([abs(x) for x in w_m])
308                    )
309
310                err_m = min(err_m, err_bound)
311
312                alpha_m = self.learning_rate * log(
313                    (self.n_classes - 1) * (1 - err_m) / err_m
314                )
315
316                self.alpha_.append(alpha_m)
317
318                w_m_temp = [exp(alpha_m * cond[i]) for i in x_range_n]
319
320                sum_w_m = sum(w_m_temp)
321
322                w_m = np.asarray([w_m_temp[i] / sum_w_m for i in x_range_n])
323
324                base_learner.set_params(seed=self.seed + (m + 1) * 1000)
325
326                if self.verbose == 1:
327                    pbar.update(m)
328
329            if self.verbose == 1:
330                pbar.update(self.n_estimators)
331
332            self.n_estimators = len(self.base_learners_)
333            self.classes_ = np.unique(y)
334
335            return self
336
337        if self.method == "SAMME.R":
338            Y = mo.one_hot_encode2(y, self.n_classes)
339
340            if sample_weight is None:
341                w_m = np.repeat(1.0 / n, n)  # (N, 1)
342
343            else:
344                w_m = np.asarray(sample_weight)
345
346            for m in range(self.n_estimators):
347                probs = base_learner.fit(
348                    X, y, sample_weight=np.ravel(w_m, order="C"), **kwargs
349                ).predict_proba(X)
350
351                np.clip(
352                    a=probs, a_min=2.220446049250313e-16, a_max=1.0, out=probs
353                )
354
355                self.base_learners_.update(
356                    {m: pickle.loads(pickle.dumps(base_learner, -1))}
357                )
358
359                w_m *= np.exp(
360                    -1.0
361                    * self.learning_rate
362                    * (1.0 - 1.0 / self.n_classes)
363                    * xlogy(Y, probs).sum(axis=1)
364                )
365
366                w_m /= np.sum(w_m)
367
368                base_learner.set_params(seed=self.seed + (m + 1) * 1000)
369
370                if self.verbose == 1:
371                    pbar.update(m)
372
373            if self.verbose == 1:
374                pbar.update(self.n_estimators)
375
376            self.n_estimators = len(self.base_learners_)
377            self.classes_ = np.unique(y)
378
379            return self

Fit Boosting model to training data (X, y).

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set or self.obj.fit

Returns:

 self: object
def predict(self, X, **kwargs):
381    def predict(self, X, **kwargs):
382        """Predict test data X.
383
384        Parameters:
385
386            X: {array-like}, shape = [n_samples, n_features]
387                Training vectors, where n_samples is the number
388                of samples and n_features is the number of features.
389
390            **kwargs: additional parameters to be passed to
391                  self.cook_test_set
392
393        Returns:
394
395            model predictions: {array-like}
396        """
397        return self.predict_proba(X, **kwargs).argmax(axis=1)

Predict test data X.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
      self.cook_test_set

Returns:

model predictions: {array-like}
def predict_proba(self, X, **kwargs):
399    def predict_proba(self, X, **kwargs):
400        """Predict probabilities for test data X.
401
402        Parameters:
403
404            X: {array-like}, shape = [n_samples, n_features]
405                Training vectors, where n_samples is the number
406                of samples and n_features is the number of features.
407
408            **kwargs: additional parameters to be passed to
409                  self.cook_test_set
410
411        Returns:
412
413            probability estimates for test data: {array-like}
414
415        """
416
417        n_iter = len(self.base_learners_)
418
419        if self.method == "SAMME":
420            ensemble_learner = np.zeros((X.shape[0], self.n_classes))
421
422            # if self.verbose == 1:
423            #    pbar = Progbar(n_iter)
424
425            for idx, base_learner in self.base_learners_.items():
426                preds = base_learner.predict(X, **kwargs)
427
428                ensemble_learner += self.alpha_[idx] * mo.one_hot_encode2(
429                    preds, self.n_classes
430                )
431
432                # if self.verbose == 1:
433                #    pbar.update(idx)
434
435            # if self.verbose == 1:
436            #    pbar.update(n_iter)
437
438            expit_ensemble_learner = expit(ensemble_learner)
439
440            sum_ensemble = expit_ensemble_learner.sum(axis=1)
441
442            return expit_ensemble_learner / sum_ensemble[:, None]
443
444        # if self.method == "SAMME.R":
445        ensemble_learner = 0
446
447        # if self.verbose == 1:
448        #    pbar = Progbar(n_iter)
449
450        for idx, base_learner in self.base_learners_.items():
451            probs = base_learner.predict_proba(X, **kwargs)
452
453            np.clip(a=probs, a_min=2.220446049250313e-16, a_max=1.0, out=probs)
454
455            log_preds_proba = np.log(probs)
456
457            ensemble_learner += (
458                log_preds_proba - log_preds_proba.mean(axis=1)[:, None]
459            )
460
461            # if self.verbose == 1:
462            #    pbar.update(idx)
463
464        ensemble_learner *= self.n_classes - 1
465
466        # if self.verbose == 1:
467        #    pbar.update(n_iter)
468
469        expit_ensemble_learner = expit(ensemble_learner)
470
471        sum_ensemble = expit_ensemble_learner.sum(axis=1)
472
473        return expit_ensemble_learner / sum_ensemble[:, None]

Predict probabilities for test data X.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
      self.cook_test_set

Returns:

probability estimates for test data: {array-like}
class Base(sklearn.base.BaseEstimator):
 34class Base(BaseEstimator):
 35    """Base model from which all the other classes inherit.
 36
 37    This class contains the most important data preprocessing/feature engineering methods.
 38
 39    Parameters:
 40
 41        n_hidden_features: int
 42            number of nodes in the hidden layer
 43
 44        activation_name: str
 45            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 46
 47        a: float
 48            hyperparameter for 'prelu' or 'elu' activation function
 49
 50        nodes_sim: str
 51            type of simulation for hidden layer nodes: 'sobol', 'hammersley', 'halton',
 52            'uniform'
 53
 54        bias: boolean
 55            indicates if the hidden layer contains a bias term (True) or
 56            not (False)
 57
 58        dropout: float
 59            regularization parameter; (random) percentage of nodes dropped out
 60            of the training
 61
 62        direct_link: boolean
 63            indicates if the original features are included (True) in model's
 64            fitting or not (False)
 65
 66        n_clusters: int
 67            number of clusters for type_clust='kmeans' or type_clust='gmm'
 68            clustering (could be 0: no clustering)
 69
 70        cluster_encode: bool
 71            defines how the variable containing clusters is treated (default is one-hot);
 72            if `False`, then labels are used, without one-hot encoding
 73
 74        type_clust: str
 75            type of clustering method: currently k-means ('kmeans') or Gaussian
 76            Mixture Model ('gmm')
 77
 78        type_scaling: a tuple of 3 strings
 79            scaling methods for inputs, hidden layer, and clustering respectively
 80            (and when relevant).
 81            Currently available: standardization ('std') or MinMax scaling ('minmax') or robust scaling ('robust') or  max absolute scaling ('maxabs')
 82
 83        col_sample: float
 84            percentage of features randomly chosen for training
 85
 86        row_sample: float
 87            percentage of rows chosen for training, by stratified bootstrapping
 88
 89        seed: int
 90            reproducibility seed for nodes_sim=='uniform', clustering and dropout
 91
 92        backend: str
 93            "cpu" or "gpu" or "tpu"
 94
 95    """
 96
 97    # construct the object -----
 98
 99    def __init__(
100        self,
101        n_hidden_features=5,
102        activation_name="relu",
103        a=0.01,
104        nodes_sim="sobol",
105        bias=True,
106        dropout=0,
107        direct_link=True,
108        n_clusters=2,
109        cluster_encode=True,
110        type_clust="kmeans",
111        type_scaling=("std", "std", "std"),
112        col_sample=1,
113        row_sample=1,
114        seed=123,
115        backend="cpu",
116    ):
117        # input checks -----
118
119        sys_platform = platform.system()
120
121        if (sys_platform == "Windows") and (backend in ("gpu", "tpu")):
122            warnings.warn(
123                "No GPU/TPU computing on Windows yet, backend set to 'cpu'"
124            )
125            backend = "cpu"
126
127        assert activation_name in (
128            "relu",
129            "tanh",
130            "sigmoid",
131            "prelu",
132            "elu",
133        ), "'activation_name' must be in ('relu', 'tanh', 'sigmoid','prelu', 'elu')"
134
135        assert nodes_sim in (
136            "sobol",
137            "hammersley",
138            "uniform",
139            "halton",
140        ), "'nodes_sim' must be in ('sobol', 'hammersley', 'uniform', 'halton')"
141
142        assert type_clust in (
143            "kmeans",
144            "gmm",
145        ), "'type_clust' must be in ('kmeans', 'gmm')"
146
147        assert (len(type_scaling) == 3) & all(
148            type_scaling[i] in ("minmax", "std", "robust", "maxabs")
149            for i in range(len(type_scaling))
150        ), "'type_scaling' must have length 3, and available scaling methods are 'minmax' scaling, standardization ('std'), robust scaling ('robust') and max absolute ('maxabs')"
151
152        assert (col_sample >= 0) & (
153            col_sample <= 1
154        ), "'col_sample' must be comprised between 0 and 1 (both included)"
155
156        assert backend in (
157            "cpu",
158            "gpu",
159            "tpu",
160        ), "must have 'backend' in ('cpu', 'gpu', 'tpu')"
161
162        self.n_hidden_features = n_hidden_features
163        self.activation_name = activation_name
164        self.a = a
165        self.nodes_sim = nodes_sim
166        self.bias = bias
167        self.seed = seed
168        self.backend = backend
169        self.dropout = dropout
170        self.direct_link = direct_link
171        self.cluster_encode = cluster_encode
172        self.type_clust = type_clust
173        self.type_scaling = type_scaling
174        self.col_sample = col_sample
175        self.row_sample = row_sample
176        self.n_clusters = n_clusters
177        if isinstance(self, RegressorMixin):
178            self.type_fit = "regression"
179        elif isinstance(self, ClassifierMixin):
180            self.type_fit = "classification"
181        self.subsampler_ = None
182        self.index_col_ = None
183        self.index_row_ = True
184        self.clustering_obj_ = None
185        self.clustering_scaler_ = None
186        self.nn_scaler_ = None
187        self.scaler_ = None
188        self.encoder_ = None
189        self.W_ = None
190        self.X_ = None
191        self.y_ = None
192        self.y_mean_ = None
193        self.beta_ = None
194
195        # activation function -----
196        if sys_platform in ("Linux", "Darwin"):
197            activation_options = {
198                "relu": ac.relu if (self.backend == "cpu") else jnn.relu,
199                "tanh": np.tanh if (self.backend == "cpu") else jnp.tanh,
200                "sigmoid": (
201                    ac.sigmoid if (self.backend == "cpu") else jnn.sigmoid
202                ),
203                "prelu": partial(ac.prelu, a=a),
204                "elu": (
205                    partial(ac.elu, a=a)
206                    if (self.backend == "cpu")
207                    else partial(jnn.elu, a=a)
208                ),
209            }
210        else:  # on Windows currently, no JAX
211            activation_options = {
212                "relu": (
213                    ac.relu if (self.backend == "cpu") else NotImplementedError
214                ),
215                "tanh": (
216                    np.tanh if (self.backend == "cpu") else NotImplementedError
217                ),
218                "sigmoid": (
219                    ac.sigmoid
220                    if (self.backend == "cpu")
221                    else NotImplementedError
222                ),
223                "prelu": partial(ac.prelu, a=a),
224                "elu": (
225                    partial(ac.elu, a=a)
226                    if (self.backend == "cpu")
227                    else NotImplementedError
228                ),
229            }
230        self.activation_func = activation_options[activation_name]
231
232    # "preprocessing" methods to be inherited -----
233
234    def encode_clusters(self, X=None, predict=False, **kwargs):  #
235        """Create new covariates with kmeans or GMM clustering
236
237        Parameters:
238
239            X: {array-like}, shape = [n_samples, n_features]
240                Training vectors, where n_samples is the number
241                of samples and n_features is the number of features.
242
243            predict: boolean
244                is False on training set and True on test set
245
246            **kwargs:
247                additional parameters to be passed to the
248                clustering method
249
250        Returns:
251
252            Clusters' matrix, one-hot encoded: {array-like}
253
254        """
255
256        np.random.seed(self.seed)
257
258        if X is None:
259            X = self.X_
260
261        if isinstance(X, pd.DataFrame):
262            X = copy.deepcopy(X.values.astype(float))
263
264        if predict is False:  # encode training set
265            # scale input data before clustering
266            self.clustering_scaler_, scaled_X = mo.scale_covariates(
267                X, choice=self.type_scaling[2]
268            )
269
270            self.clustering_obj_, X_clustered = mo.cluster_covariates(
271                scaled_X,
272                self.n_clusters,
273                self.seed,
274                type_clust=self.type_clust,
275                **kwargs
276            )
277
278            if self.cluster_encode == True:
279                return mo.one_hot_encode(X_clustered, self.n_clusters).astype(
280                    np.float16
281                )
282
283            return X_clustered.astype(np.float16)
284
285        # if predict == True, encode test set
286        X_clustered = self.clustering_obj_.predict(
287            self.clustering_scaler_.transform(X)
288        )
289
290        if self.cluster_encode == True:
291            return mo.one_hot_encode(X_clustered, self.n_clusters).astype(
292                np.float16
293            )
294
295        return X_clustered.astype(np.float16)
296
297    def create_layer(self, scaled_X, W=None):
298        """Create hidden layer.
299
300        Parameters:
301
302            scaled_X: {array-like}, shape = [n_samples, n_features]
303                Training vectors, where n_samples is the number
304                of samples and n_features is the number of features
305
306            W: {array-like}, shape = [n_features, hidden_features]
307                if provided, constructs the hidden layer with W; otherwise computed internally
308
309        Returns:
310
311            Hidden layer matrix: {array-like}
312
313        """
314
315        n_features = scaled_X.shape[1]
316
317        # hash_sim = {
318        #         "sobol": generate_sobol,
319        #         "hammersley": generate_hammersley,
320        #         "uniform": generate_uniform,
321        #         "halton": generate_halton
322        #     }
323
324        if self.bias is False:  # no bias term in the hidden layer
325            if W is None:
326                if self.nodes_sim == "sobol":
327                    self.W_ = generate_sobol(
328                        n_dims=n_features,
329                        n_points=self.n_hidden_features,
330                        seed=self.seed,
331                    )
332                elif self.nodes_sim == "hammersley":
333                    self.W_ = generate_hammersley(
334                        n_dims=n_features,
335                        n_points=self.n_hidden_features,
336                        seed=self.seed,
337                    )
338                elif self.nodes_sim == "uniform":
339                    self.W_ = generate_uniform(
340                        n_dims=n_features,
341                        n_points=self.n_hidden_features,
342                        seed=self.seed,
343                    )
344                else:
345                    self.W_ = generate_halton(
346                        n_dims=n_features,
347                        n_points=self.n_hidden_features,
348                        seed=self.seed,
349                    )
350
351                # self.W_ = hash_sim[self.nodes_sim](
352                #             n_dims=n_features,
353                #             n_points=self.n_hidden_features,
354                #             seed=self.seed,
355                #         )
356
357                assert (
358                    scaled_X.shape[1] == self.W_.shape[0]
359                ), "check dimensions of covariates X and matrix W"
360
361                return mo.dropout(
362                    x=self.activation_func(
363                        mo.safe_sparse_dot(
364                            a=scaled_X, b=self.W_, backend=self.backend
365                        )
366                    ),
367                    drop_prob=self.dropout,
368                    seed=self.seed,
369                )
370
371            # W is not none
372            assert (
373                scaled_X.shape[1] == W.shape[0]
374            ), "check dimensions of covariates X and matrix W"
375
376            # self.W_ = W
377            return mo.dropout(
378                x=self.activation_func(
379                    mo.safe_sparse_dot(a=scaled_X, b=W, backend=self.backend)
380                ),
381                drop_prob=self.dropout,
382                seed=self.seed,
383            )
384
385        # with bias term in the hidden layer
386        if W is None:
387            n_features_1 = n_features + 1
388
389            if self.nodes_sim == "sobol":
390                self.W_ = generate_sobol(
391                    n_dims=n_features_1,
392                    n_points=self.n_hidden_features,
393                    seed=self.seed,
394                )
395            elif self.nodes_sim == "hammersley":
396                self.W_ = generate_hammersley(
397                    n_dims=n_features_1,
398                    n_points=self.n_hidden_features,
399                    seed=self.seed,
400                )
401            elif self.nodes_sim == "uniform":
402                self.W_ = generate_uniform(
403                    n_dims=n_features_1,
404                    n_points=self.n_hidden_features,
405                    seed=self.seed,
406                )
407            else:
408                self.W_ = generate_halton(
409                    n_dims=n_features_1,
410                    n_points=self.n_hidden_features,
411                    seed=self.seed,
412                )
413
414            # self.W_ = hash_sim[self.nodes_sim](
415            #         n_dims=n_features_1,
416            #         n_points=self.n_hidden_features,
417            #         seed=self.seed,
418            #     )
419
420            return mo.dropout(
421                x=self.activation_func(
422                    mo.safe_sparse_dot(
423                        a=mo.cbind(
424                            np.ones(scaled_X.shape[0]),
425                            scaled_X,
426                            backend=self.backend,
427                        ),
428                        b=self.W_,
429                        backend=self.backend,
430                    )
431                ),
432                drop_prob=self.dropout,
433                seed=self.seed,
434            )
435
436        # W is not None
437        # self.W_ = W
438        return mo.dropout(
439            x=self.activation_func(
440                mo.safe_sparse_dot(
441                    a=mo.cbind(
442                        np.ones(scaled_X.shape[0]),
443                        scaled_X,
444                        backend=self.backend,
445                    ),
446                    b=W,
447                    backend=self.backend,
448                )
449            ),
450            drop_prob=self.dropout,
451            seed=self.seed,
452        )
453
454    def cook_training_set(self, y=None, X=None, W=None, **kwargs):
455        """Create new hidden features for training set, with hidden layer, center the response.
456
457        Parameters:
458
459            y: array-like, shape = [n_samples]
460                Target values
461
462            X: {array-like}, shape = [n_samples, n_features]
463                Training vectors, where n_samples is the number
464                of samples and n_features is the number of features
465
466            W: {array-like}, shape = [n_features, hidden_features]
467                if provided, constructs the hidden layer via W
468
469        Returns:
470
471            (centered response, direct link + hidden layer matrix): {tuple}
472
473        """
474
475        # either X and y are stored or not
476        # assert ((y is None) & (X is None)) | ((y is not None) & (X is not None))
477        if self.n_hidden_features > 0:  # has a hidden layer
478            assert (
479                len(self.type_scaling) >= 2
480            ), "must have len(self.type_scaling) >= 2 when self.n_hidden_features > 0"
481
482        if X is None:
483            if self.col_sample == 1:
484                input_X = self.X_
485            else:
486                n_features = self.X_.shape[1]
487                new_n_features = int(np.ceil(n_features * self.col_sample))
488                assert (
489                    new_n_features >= 1
490                ), "check class attribute 'col_sample' and the number of covariates provided for X"
491                np.random.seed(self.seed)
492                index_col = np.random.choice(
493                    range(n_features), size=new_n_features, replace=False
494                )
495                self.index_col_ = index_col
496                input_X = self.X_[:, self.index_col_]
497
498        else:  # X is not None # keep X vs self.X_
499            if isinstance(X, pd.DataFrame):
500                X = copy.deepcopy(X.values.astype(float))
501
502            if self.col_sample == 1:
503                input_X = X
504            else:
505                n_features = X.shape[1]
506                new_n_features = int(np.ceil(n_features * self.col_sample))
507                assert (
508                    new_n_features >= 1
509                ), "check class attribute 'col_sample' and the number of covariates provided for X"
510                np.random.seed(self.seed)
511                index_col = np.random.choice(
512                    range(n_features), size=new_n_features, replace=False
513                )
514                self.index_col_ = index_col
515                input_X = X[:, self.index_col_]
516
517        if (
518            self.n_clusters <= 0
519        ):  # data without any clustering: self.n_clusters is None -----
520            if self.n_hidden_features > 0:  # with hidden layer
521                self.nn_scaler_, scaled_X = mo.scale_covariates(
522                    input_X, choice=self.type_scaling[1]
523                )
524                Phi_X = (
525                    self.create_layer(scaled_X)
526                    if W is None
527                    else self.create_layer(scaled_X, W=W)
528                )
529                Z = (
530                    mo.cbind(input_X, Phi_X, backend=self.backend)
531                    if self.direct_link is True
532                    else Phi_X
533                )
534                self.scaler_, scaled_Z = mo.scale_covariates(
535                    Z, choice=self.type_scaling[0]
536                )
537            else:  # no hidden layer
538                Z = input_X
539                self.scaler_, scaled_Z = mo.scale_covariates(
540                    Z, choice=self.type_scaling[0]
541                )
542        else:  # data with clustering: self.n_clusters is not None ----- # keep
543            augmented_X = mo.cbind(
544                input_X,
545                self.encode_clusters(input_X, **kwargs),
546                backend=self.backend,
547            )
548
549            if self.n_hidden_features > 0:  # with hidden layer
550                self.nn_scaler_, scaled_X = mo.scale_covariates(
551                    augmented_X, choice=self.type_scaling[1]
552                )
553                Phi_X = (
554                    self.create_layer(scaled_X)
555                    if W is None
556                    else self.create_layer(scaled_X, W=W)
557                )
558                Z = (
559                    mo.cbind(augmented_X, Phi_X, backend=self.backend)
560                    if self.direct_link is True
561                    else Phi_X
562                )
563                self.scaler_, scaled_Z = mo.scale_covariates(
564                    Z, choice=self.type_scaling[0]
565                )
566            else:  # no hidden layer
567                Z = augmented_X
568                self.scaler_, scaled_Z = mo.scale_covariates(
569                    Z, choice=self.type_scaling[0]
570                )
571
572        # Returning model inputs -----
573        if mx.is_factor(y) is False:  # regression
574            # center y
575            if y is None:
576                self.y_mean_, centered_y = mo.center_response(self.y_)
577            else:
578                self.y_mean_, centered_y = mo.center_response(y)
579
580            # y is subsampled
581            if self.row_sample < 1:
582                n, p = Z.shape
583
584                self.subsampler_ = (
585                    SubSampler(
586                        y=self.y_, row_sample=self.row_sample, seed=self.seed
587                    )
588                    if y is None
589                    else SubSampler(
590                        y=y, row_sample=self.row_sample, seed=self.seed
591                    )
592                )
593
594                self.index_row_ = self.subsampler_.subsample()
595
596                n_row_sample = len(self.index_row_)
597                # regression
598                return (
599                    centered_y[self.index_row_].reshape(n_row_sample),
600                    self.scaler_.transform(
601                        Z[self.index_row_, :].reshape(n_row_sample, p)
602                    ),
603                )
604            # y is not subsampled
605            # regression
606            return (centered_y, self.scaler_.transform(Z))
607
608        # classification
609        # y is subsampled
610        if self.row_sample < 1:
611            n, p = Z.shape
612
613            self.subsampler_ = (
614                SubSampler(
615                    y=self.y_, row_sample=self.row_sample, seed=self.seed
616                )
617                if y is None
618                else SubSampler(y=y, row_sample=self.row_sample, seed=self.seed)
619            )
620
621            self.index_row_ = self.subsampler_.subsample()
622
623            n_row_sample = len(self.index_row_)
624            # classification
625            return (
626                y[self.index_row_].reshape(n_row_sample),
627                self.scaler_.transform(
628                    Z[self.index_row_, :].reshape(n_row_sample, p)
629                ),
630            )
631        # y is not subsampled
632        # classification
633        return (y, self.scaler_.transform(Z))
634
635    def cook_test_set(self, X, **kwargs):
636        """Transform data from test set, with hidden layer.
637
638        Parameters:
639
640            X: {array-like}, shape = [n_samples, n_features]
641                Training vectors, where n_samples is the number
642                of samples and n_features is the number of features
643
644            **kwargs: additional parameters to be passed to self.encode_cluster
645
646        Returns:
647
648            Transformed test set : {array-like}
649        """
650
651        if isinstance(X, pd.DataFrame):
652            X = copy.deepcopy(X.values.astype(float))
653
654        if (
655            self.n_clusters == 0
656        ):  # data without clustering: self.n_clusters is None -----
657            if self.n_hidden_features > 0:
658                # if hidden layer
659                scaled_X = (
660                    self.nn_scaler_.transform(X)
661                    if (self.col_sample == 1)
662                    else self.nn_scaler_.transform(X[:, self.index_col_])
663                )
664                Phi_X = self.create_layer(scaled_X, self.W_)
665                if self.direct_link == True:
666                    return self.scaler_.transform(
667                        mo.cbind(scaled_X, Phi_X, backend=self.backend)
668                    )
669                # when self.direct_link == False
670                return self.scaler_.transform(Phi_X)
671            # if no hidden layer # self.n_hidden_features == 0
672            return self.scaler_.transform(X)
673
674        # data with clustering: self.n_clusters > 0 -----
675        if self.col_sample == 1:
676            predicted_clusters = self.encode_clusters(
677                X=X, predict=True, **kwargs
678            )
679            augmented_X = mo.cbind(X, predicted_clusters, backend=self.backend)
680        else:
681            predicted_clusters = self.encode_clusters(
682                X=X[:, self.index_col_], predict=True, **kwargs
683            )
684            augmented_X = mo.cbind(
685                X[:, self.index_col_], predicted_clusters, backend=self.backend
686            )
687
688        if self.n_hidden_features > 0:  # if hidden layer
689            scaled_X = self.nn_scaler_.transform(augmented_X)
690            Phi_X = self.create_layer(scaled_X, self.W_)
691            if self.direct_link == True:
692                return self.scaler_.transform(
693                    mo.cbind(augmented_X, Phi_X, backend=self.backend)
694                )
695            return self.scaler_.transform(Phi_X)
696
697        # if no hidden layer
698        return self.scaler_.transform(augmented_X)
699
700    def score(self, X, y, scoring=None, **kwargs):
701        """Score the model on test set features X and response y.
702
703        Parameters:
704
705            X: {array-like}, shape = [n_samples, n_features]
706                Training vectors, where n_samples is the number
707                of samples and n_features is the number of features
708
709            y: array-like, shape = [n_samples]
710                Target values
711
712            scoring: str
713                must be in ('explained_variance', 'neg_mean_absolute_error',
714                            'neg_mean_squared_error', 'neg_mean_squared_log_error',
715                            'neg_median_absolute_error', 'r2')
716
717            **kwargs: additional parameters to be passed to scoring functions
718
719        Returns:
720
721        model scores: {array-like}
722
723        """
724
725        preds = self.predict(X)
726
727        if self.type_fit == "classification":
728
729            if scoring is None:
730                scoring = "accuracy"
731
732            # check inputs
733            assert scoring in (
734                "accuracy",
735                "average_precision",
736                "brier_score_loss",
737                "f1",
738                "f1_micro",
739                "f1_macro",
740                "f1_weighted",
741                "f1_samples",
742                "neg_log_loss",
743                "precision",
744                "recall",
745                "roc_auc",
746            ), "'scoring' should be in ('accuracy', 'average_precision', \
747                            'brier_score_loss', 'f1', 'f1_micro', \
748                            'f1_macro', 'f1_weighted',  'f1_samples', \
749                            'neg_log_loss', 'precision', 'recall', \
750                            'roc_auc')"
751
752            scoring_options = {
753                "accuracy": skm.accuracy_score,
754                "average_precision": skm.average_precision_score,
755                "brier_score_loss": skm.brier_score_loss,
756                "f1": skm.f1_score,
757                "f1_micro": skm.f1_score,
758                "f1_macro": skm.f1_score,
759                "f1_weighted": skm.f1_score,
760                "f1_samples": skm.f1_score,
761                "neg_log_loss": skm.log_loss,
762                "precision": skm.precision_score,
763                "recall": skm.recall_score,
764                "roc_auc": skm.roc_auc_score,
765            }
766
767            return scoring_options[scoring](y, preds, **kwargs)
768
769        if self.type_fit == "regression":
770
771            if (
772                type(preds) == tuple
773            ):  # if there are std. devs in the predictions
774                preds = preds[0]
775
776            if scoring is None:
777                scoring = "neg_root_mean_squared_error"
778
779            # check inputs
780            assert scoring in (
781                "explained_variance",
782                "neg_mean_absolute_error",
783                "neg_mean_squared_error",
784                "neg_mean_squared_log_error",
785                "neg_median_absolute_error",
786                "neg_root_mean_squared_error",
787                "r2",
788            ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
789                            'neg_mean_squared_error', 'neg_mean_squared_log_error', \
790                            'neg_median_absolute_error', 'r2', 'neg_root_mean_squared_error')"
791
792            scoring_options = {
793                "neg_root_mean_squared_error": skm.root_mean_squared_error,
794                "explained_variance": skm.explained_variance_score,
795                "neg_mean_absolute_error": skm.median_absolute_error,
796                "neg_mean_squared_error": skm.mean_squared_error,
797                "neg_mean_squared_log_error": skm.mean_squared_log_error,
798                "neg_median_absolute_error": skm.median_absolute_error,
799                "r2": skm.r2_score,
800            }
801
802            return scoring_options[scoring](y, preds, **kwargs)

Base model from which all the other classes inherit.

This class contains the most important data preprocessing/feature engineering methods.

Parameters:

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for hidden layer nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or
    not (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original features are included (True) in model's
    fitting or not (False)

n_clusters: int
    number of clusters for type_clust='kmeans' or type_clust='gmm'
    clustering (could be 0: no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot);
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax') or robust scaling ('robust') or  max absolute scaling ('maxabs')

col_sample: float
    percentage of features randomly chosen for training

row_sample: float
    percentage of rows chosen for training, by stratified bootstrapping

seed: int
    reproducibility seed for nodes_sim=='uniform', clustering and dropout

backend: str
    "cpu" or "gpu" or "tpu"
def encode_clusters(self, X=None, predict=False, **kwargs):
234    def encode_clusters(self, X=None, predict=False, **kwargs):  #
235        """Create new covariates with kmeans or GMM clustering
236
237        Parameters:
238
239            X: {array-like}, shape = [n_samples, n_features]
240                Training vectors, where n_samples is the number
241                of samples and n_features is the number of features.
242
243            predict: boolean
244                is False on training set and True on test set
245
246            **kwargs:
247                additional parameters to be passed to the
248                clustering method
249
250        Returns:
251
252            Clusters' matrix, one-hot encoded: {array-like}
253
254        """
255
256        np.random.seed(self.seed)
257
258        if X is None:
259            X = self.X_
260
261        if isinstance(X, pd.DataFrame):
262            X = copy.deepcopy(X.values.astype(float))
263
264        if predict is False:  # encode training set
265            # scale input data before clustering
266            self.clustering_scaler_, scaled_X = mo.scale_covariates(
267                X, choice=self.type_scaling[2]
268            )
269
270            self.clustering_obj_, X_clustered = mo.cluster_covariates(
271                scaled_X,
272                self.n_clusters,
273                self.seed,
274                type_clust=self.type_clust,
275                **kwargs
276            )
277
278            if self.cluster_encode == True:
279                return mo.one_hot_encode(X_clustered, self.n_clusters).astype(
280                    np.float16
281                )
282
283            return X_clustered.astype(np.float16)
284
285        # if predict == True, encode test set
286        X_clustered = self.clustering_obj_.predict(
287            self.clustering_scaler_.transform(X)
288        )
289
290        if self.cluster_encode == True:
291            return mo.one_hot_encode(X_clustered, self.n_clusters).astype(
292                np.float16
293            )
294
295        return X_clustered.astype(np.float16)

Create new covariates with kmeans or GMM clustering

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

predict: boolean
    is False on training set and True on test set

**kwargs:
    additional parameters to be passed to the
    clustering method

Returns:

Clusters' matrix, one-hot encoded: {array-like}
def create_layer(self, scaled_X, W=None):
297    def create_layer(self, scaled_X, W=None):
298        """Create hidden layer.
299
300        Parameters:
301
302            scaled_X: {array-like}, shape = [n_samples, n_features]
303                Training vectors, where n_samples is the number
304                of samples and n_features is the number of features
305
306            W: {array-like}, shape = [n_features, hidden_features]
307                if provided, constructs the hidden layer with W; otherwise computed internally
308
309        Returns:
310
311            Hidden layer matrix: {array-like}
312
313        """
314
315        n_features = scaled_X.shape[1]
316
317        # hash_sim = {
318        #         "sobol": generate_sobol,
319        #         "hammersley": generate_hammersley,
320        #         "uniform": generate_uniform,
321        #         "halton": generate_halton
322        #     }
323
324        if self.bias is False:  # no bias term in the hidden layer
325            if W is None:
326                if self.nodes_sim == "sobol":
327                    self.W_ = generate_sobol(
328                        n_dims=n_features,
329                        n_points=self.n_hidden_features,
330                        seed=self.seed,
331                    )
332                elif self.nodes_sim == "hammersley":
333                    self.W_ = generate_hammersley(
334                        n_dims=n_features,
335                        n_points=self.n_hidden_features,
336                        seed=self.seed,
337                    )
338                elif self.nodes_sim == "uniform":
339                    self.W_ = generate_uniform(
340                        n_dims=n_features,
341                        n_points=self.n_hidden_features,
342                        seed=self.seed,
343                    )
344                else:
345                    self.W_ = generate_halton(
346                        n_dims=n_features,
347                        n_points=self.n_hidden_features,
348                        seed=self.seed,
349                    )
350
351                # self.W_ = hash_sim[self.nodes_sim](
352                #             n_dims=n_features,
353                #             n_points=self.n_hidden_features,
354                #             seed=self.seed,
355                #         )
356
357                assert (
358                    scaled_X.shape[1] == self.W_.shape[0]
359                ), "check dimensions of covariates X and matrix W"
360
361                return mo.dropout(
362                    x=self.activation_func(
363                        mo.safe_sparse_dot(
364                            a=scaled_X, b=self.W_, backend=self.backend
365                        )
366                    ),
367                    drop_prob=self.dropout,
368                    seed=self.seed,
369                )
370
371            # W is not none
372            assert (
373                scaled_X.shape[1] == W.shape[0]
374            ), "check dimensions of covariates X and matrix W"
375
376            # self.W_ = W
377            return mo.dropout(
378                x=self.activation_func(
379                    mo.safe_sparse_dot(a=scaled_X, b=W, backend=self.backend)
380                ),
381                drop_prob=self.dropout,
382                seed=self.seed,
383            )
384
385        # with bias term in the hidden layer
386        if W is None:
387            n_features_1 = n_features + 1
388
389            if self.nodes_sim == "sobol":
390                self.W_ = generate_sobol(
391                    n_dims=n_features_1,
392                    n_points=self.n_hidden_features,
393                    seed=self.seed,
394                )
395            elif self.nodes_sim == "hammersley":
396                self.W_ = generate_hammersley(
397                    n_dims=n_features_1,
398                    n_points=self.n_hidden_features,
399                    seed=self.seed,
400                )
401            elif self.nodes_sim == "uniform":
402                self.W_ = generate_uniform(
403                    n_dims=n_features_1,
404                    n_points=self.n_hidden_features,
405                    seed=self.seed,
406                )
407            else:
408                self.W_ = generate_halton(
409                    n_dims=n_features_1,
410                    n_points=self.n_hidden_features,
411                    seed=self.seed,
412                )
413
414            # self.W_ = hash_sim[self.nodes_sim](
415            #         n_dims=n_features_1,
416            #         n_points=self.n_hidden_features,
417            #         seed=self.seed,
418            #     )
419
420            return mo.dropout(
421                x=self.activation_func(
422                    mo.safe_sparse_dot(
423                        a=mo.cbind(
424                            np.ones(scaled_X.shape[0]),
425                            scaled_X,
426                            backend=self.backend,
427                        ),
428                        b=self.W_,
429                        backend=self.backend,
430                    )
431                ),
432                drop_prob=self.dropout,
433                seed=self.seed,
434            )
435
436        # W is not None
437        # self.W_ = W
438        return mo.dropout(
439            x=self.activation_func(
440                mo.safe_sparse_dot(
441                    a=mo.cbind(
442                        np.ones(scaled_X.shape[0]),
443                        scaled_X,
444                        backend=self.backend,
445                    ),
446                    b=W,
447                    backend=self.backend,
448                )
449            ),
450            drop_prob=self.dropout,
451            seed=self.seed,
452        )

Create hidden layer.

Parameters:

scaled_X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features

W: {array-like}, shape = [n_features, hidden_features]
    if provided, constructs the hidden layer with W; otherwise computed internally

Returns:

Hidden layer matrix: {array-like}
def cook_training_set(self, y=None, X=None, W=None, **kwargs):
454    def cook_training_set(self, y=None, X=None, W=None, **kwargs):
455        """Create new hidden features for training set, with hidden layer, center the response.
456
457        Parameters:
458
459            y: array-like, shape = [n_samples]
460                Target values
461
462            X: {array-like}, shape = [n_samples, n_features]
463                Training vectors, where n_samples is the number
464                of samples and n_features is the number of features
465
466            W: {array-like}, shape = [n_features, hidden_features]
467                if provided, constructs the hidden layer via W
468
469        Returns:
470
471            (centered response, direct link + hidden layer matrix): {tuple}
472
473        """
474
475        # either X and y are stored or not
476        # assert ((y is None) & (X is None)) | ((y is not None) & (X is not None))
477        if self.n_hidden_features > 0:  # has a hidden layer
478            assert (
479                len(self.type_scaling) >= 2
480            ), "must have len(self.type_scaling) >= 2 when self.n_hidden_features > 0"
481
482        if X is None:
483            if self.col_sample == 1:
484                input_X = self.X_
485            else:
486                n_features = self.X_.shape[1]
487                new_n_features = int(np.ceil(n_features * self.col_sample))
488                assert (
489                    new_n_features >= 1
490                ), "check class attribute 'col_sample' and the number of covariates provided for X"
491                np.random.seed(self.seed)
492                index_col = np.random.choice(
493                    range(n_features), size=new_n_features, replace=False
494                )
495                self.index_col_ = index_col
496                input_X = self.X_[:, self.index_col_]
497
498        else:  # X is not None # keep X vs self.X_
499            if isinstance(X, pd.DataFrame):
500                X = copy.deepcopy(X.values.astype(float))
501
502            if self.col_sample == 1:
503                input_X = X
504            else:
505                n_features = X.shape[1]
506                new_n_features = int(np.ceil(n_features * self.col_sample))
507                assert (
508                    new_n_features >= 1
509                ), "check class attribute 'col_sample' and the number of covariates provided for X"
510                np.random.seed(self.seed)
511                index_col = np.random.choice(
512                    range(n_features), size=new_n_features, replace=False
513                )
514                self.index_col_ = index_col
515                input_X = X[:, self.index_col_]
516
517        if (
518            self.n_clusters <= 0
519        ):  # data without any clustering: self.n_clusters is None -----
520            if self.n_hidden_features > 0:  # with hidden layer
521                self.nn_scaler_, scaled_X = mo.scale_covariates(
522                    input_X, choice=self.type_scaling[1]
523                )
524                Phi_X = (
525                    self.create_layer(scaled_X)
526                    if W is None
527                    else self.create_layer(scaled_X, W=W)
528                )
529                Z = (
530                    mo.cbind(input_X, Phi_X, backend=self.backend)
531                    if self.direct_link is True
532                    else Phi_X
533                )
534                self.scaler_, scaled_Z = mo.scale_covariates(
535                    Z, choice=self.type_scaling[0]
536                )
537            else:  # no hidden layer
538                Z = input_X
539                self.scaler_, scaled_Z = mo.scale_covariates(
540                    Z, choice=self.type_scaling[0]
541                )
542        else:  # data with clustering: self.n_clusters is not None ----- # keep
543            augmented_X = mo.cbind(
544                input_X,
545                self.encode_clusters(input_X, **kwargs),
546                backend=self.backend,
547            )
548
549            if self.n_hidden_features > 0:  # with hidden layer
550                self.nn_scaler_, scaled_X = mo.scale_covariates(
551                    augmented_X, choice=self.type_scaling[1]
552                )
553                Phi_X = (
554                    self.create_layer(scaled_X)
555                    if W is None
556                    else self.create_layer(scaled_X, W=W)
557                )
558                Z = (
559                    mo.cbind(augmented_X, Phi_X, backend=self.backend)
560                    if self.direct_link is True
561                    else Phi_X
562                )
563                self.scaler_, scaled_Z = mo.scale_covariates(
564                    Z, choice=self.type_scaling[0]
565                )
566            else:  # no hidden layer
567                Z = augmented_X
568                self.scaler_, scaled_Z = mo.scale_covariates(
569                    Z, choice=self.type_scaling[0]
570                )
571
572        # Returning model inputs -----
573        if mx.is_factor(y) is False:  # regression
574            # center y
575            if y is None:
576                self.y_mean_, centered_y = mo.center_response(self.y_)
577            else:
578                self.y_mean_, centered_y = mo.center_response(y)
579
580            # y is subsampled
581            if self.row_sample < 1:
582                n, p = Z.shape
583
584                self.subsampler_ = (
585                    SubSampler(
586                        y=self.y_, row_sample=self.row_sample, seed=self.seed
587                    )
588                    if y is None
589                    else SubSampler(
590                        y=y, row_sample=self.row_sample, seed=self.seed
591                    )
592                )
593
594                self.index_row_ = self.subsampler_.subsample()
595
596                n_row_sample = len(self.index_row_)
597                # regression
598                return (
599                    centered_y[self.index_row_].reshape(n_row_sample),
600                    self.scaler_.transform(
601                        Z[self.index_row_, :].reshape(n_row_sample, p)
602                    ),
603                )
604            # y is not subsampled
605            # regression
606            return (centered_y, self.scaler_.transform(Z))
607
608        # classification
609        # y is subsampled
610        if self.row_sample < 1:
611            n, p = Z.shape
612
613            self.subsampler_ = (
614                SubSampler(
615                    y=self.y_, row_sample=self.row_sample, seed=self.seed
616                )
617                if y is None
618                else SubSampler(y=y, row_sample=self.row_sample, seed=self.seed)
619            )
620
621            self.index_row_ = self.subsampler_.subsample()
622
623            n_row_sample = len(self.index_row_)
624            # classification
625            return (
626                y[self.index_row_].reshape(n_row_sample),
627                self.scaler_.transform(
628                    Z[self.index_row_, :].reshape(n_row_sample, p)
629                ),
630            )
631        # y is not subsampled
632        # classification
633        return (y, self.scaler_.transform(Z))

Create new hidden features for training set, with hidden layer, center the response.

Parameters:

y: array-like, shape = [n_samples]
    Target values

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features

W: {array-like}, shape = [n_features, hidden_features]
    if provided, constructs the hidden layer via W

Returns:

(centered response, direct link + hidden layer matrix): {tuple}
def cook_test_set(self, X, **kwargs):
635    def cook_test_set(self, X, **kwargs):
636        """Transform data from test set, with hidden layer.
637
638        Parameters:
639
640            X: {array-like}, shape = [n_samples, n_features]
641                Training vectors, where n_samples is the number
642                of samples and n_features is the number of features
643
644            **kwargs: additional parameters to be passed to self.encode_cluster
645
646        Returns:
647
648            Transformed test set : {array-like}
649        """
650
651        if isinstance(X, pd.DataFrame):
652            X = copy.deepcopy(X.values.astype(float))
653
654        if (
655            self.n_clusters == 0
656        ):  # data without clustering: self.n_clusters is None -----
657            if self.n_hidden_features > 0:
658                # if hidden layer
659                scaled_X = (
660                    self.nn_scaler_.transform(X)
661                    if (self.col_sample == 1)
662                    else self.nn_scaler_.transform(X[:, self.index_col_])
663                )
664                Phi_X = self.create_layer(scaled_X, self.W_)
665                if self.direct_link == True:
666                    return self.scaler_.transform(
667                        mo.cbind(scaled_X, Phi_X, backend=self.backend)
668                    )
669                # when self.direct_link == False
670                return self.scaler_.transform(Phi_X)
671            # if no hidden layer # self.n_hidden_features == 0
672            return self.scaler_.transform(X)
673
674        # data with clustering: self.n_clusters > 0 -----
675        if self.col_sample == 1:
676            predicted_clusters = self.encode_clusters(
677                X=X, predict=True, **kwargs
678            )
679            augmented_X = mo.cbind(X, predicted_clusters, backend=self.backend)
680        else:
681            predicted_clusters = self.encode_clusters(
682                X=X[:, self.index_col_], predict=True, **kwargs
683            )
684            augmented_X = mo.cbind(
685                X[:, self.index_col_], predicted_clusters, backend=self.backend
686            )
687
688        if self.n_hidden_features > 0:  # if hidden layer
689            scaled_X = self.nn_scaler_.transform(augmented_X)
690            Phi_X = self.create_layer(scaled_X, self.W_)
691            if self.direct_link == True:
692                return self.scaler_.transform(
693                    mo.cbind(augmented_X, Phi_X, backend=self.backend)
694                )
695            return self.scaler_.transform(Phi_X)
696
697        # if no hidden layer
698        return self.scaler_.transform(augmented_X)

Transform data from test set, with hidden layer.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features

**kwargs: additional parameters to be passed to self.encode_cluster

Returns:

Transformed test set : {array-like}
def score(self, X, y, scoring=None, **kwargs):
700    def score(self, X, y, scoring=None, **kwargs):
701        """Score the model on test set features X and response y.
702
703        Parameters:
704
705            X: {array-like}, shape = [n_samples, n_features]
706                Training vectors, where n_samples is the number
707                of samples and n_features is the number of features
708
709            y: array-like, shape = [n_samples]
710                Target values
711
712            scoring: str
713                must be in ('explained_variance', 'neg_mean_absolute_error',
714                            'neg_mean_squared_error', 'neg_mean_squared_log_error',
715                            'neg_median_absolute_error', 'r2')
716
717            **kwargs: additional parameters to be passed to scoring functions
718
719        Returns:
720
721        model scores: {array-like}
722
723        """
724
725        preds = self.predict(X)
726
727        if self.type_fit == "classification":
728
729            if scoring is None:
730                scoring = "accuracy"
731
732            # check inputs
733            assert scoring in (
734                "accuracy",
735                "average_precision",
736                "brier_score_loss",
737                "f1",
738                "f1_micro",
739                "f1_macro",
740                "f1_weighted",
741                "f1_samples",
742                "neg_log_loss",
743                "precision",
744                "recall",
745                "roc_auc",
746            ), "'scoring' should be in ('accuracy', 'average_precision', \
747                            'brier_score_loss', 'f1', 'f1_micro', \
748                            'f1_macro', 'f1_weighted',  'f1_samples', \
749                            'neg_log_loss', 'precision', 'recall', \
750                            'roc_auc')"
751
752            scoring_options = {
753                "accuracy": skm.accuracy_score,
754                "average_precision": skm.average_precision_score,
755                "brier_score_loss": skm.brier_score_loss,
756                "f1": skm.f1_score,
757                "f1_micro": skm.f1_score,
758                "f1_macro": skm.f1_score,
759                "f1_weighted": skm.f1_score,
760                "f1_samples": skm.f1_score,
761                "neg_log_loss": skm.log_loss,
762                "precision": skm.precision_score,
763                "recall": skm.recall_score,
764                "roc_auc": skm.roc_auc_score,
765            }
766
767            return scoring_options[scoring](y, preds, **kwargs)
768
769        if self.type_fit == "regression":
770
771            if (
772                type(preds) == tuple
773            ):  # if there are std. devs in the predictions
774                preds = preds[0]
775
776            if scoring is None:
777                scoring = "neg_root_mean_squared_error"
778
779            # check inputs
780            assert scoring in (
781                "explained_variance",
782                "neg_mean_absolute_error",
783                "neg_mean_squared_error",
784                "neg_mean_squared_log_error",
785                "neg_median_absolute_error",
786                "neg_root_mean_squared_error",
787                "r2",
788            ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
789                            'neg_mean_squared_error', 'neg_mean_squared_log_error', \
790                            'neg_median_absolute_error', 'r2', 'neg_root_mean_squared_error')"
791
792            scoring_options = {
793                "neg_root_mean_squared_error": skm.root_mean_squared_error,
794                "explained_variance": skm.explained_variance_score,
795                "neg_mean_absolute_error": skm.median_absolute_error,
796                "neg_mean_squared_error": skm.mean_squared_error,
797                "neg_mean_squared_log_error": skm.mean_squared_log_error,
798                "neg_median_absolute_error": skm.median_absolute_error,
799                "r2": skm.r2_score,
800            }
801
802            return scoring_options[scoring](y, preds, **kwargs)

Score the model on test set features X and response y.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

scoring: str
    must be in ('explained_variance', 'neg_mean_absolute_error',
                'neg_mean_squared_error', 'neg_mean_squared_log_error',
                'neg_median_absolute_error', 'r2')

**kwargs: additional parameters to be passed to scoring functions

Returns:

model scores: {array-like}

class BaseRegressor(nnetsauce.Base, sklearn.base.RegressorMixin):
 15class BaseRegressor(Base, RegressorMixin):
 16    """Random Vector Functional Link Network regression without shrinkage
 17
 18    Parameters:
 19
 20        n_hidden_features: int
 21            number of nodes in the hidden layer
 22
 23        activation_name: str
 24            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 25
 26        a: float
 27            hyperparameter for 'prelu' or 'elu' activation function
 28
 29        nodes_sim: str
 30            type of simulation for hidden layer nodes: 'sobol', 'hammersley', 'halton',
 31            'uniform'
 32
 33        bias: boolean
 34            indicates if the hidden layer contains a bias term (True) or
 35            not (False)
 36
 37        dropout: float
 38            regularization parameter; (random) percentage of nodes dropped out
 39            of the training
 40
 41        direct_link: boolean
 42            indicates if the original features are included (True) in model's
 43            fitting or not (False)
 44
 45        n_clusters: int
 46            number of clusters for type_clust='kmeans' or type_clust='gmm'
 47            clustering (could be 0: no clustering)
 48
 49        cluster_encode: bool
 50            defines how the variable containing clusters is treated (default is one-hot);
 51            if `False`, then labels are used, without one-hot encoding
 52
 53        type_clust: str
 54            type of clustering method: currently k-means ('kmeans') or Gaussian
 55            Mixture Model ('gmm')
 56
 57        type_scaling: a tuple of 3 strings
 58            scaling methods for inputs, hidden layer, and clustering respectively
 59            (and when relevant).
 60            Currently available: standardization ('std') or MinMax scaling ('minmax')
 61
 62        col_sample: float
 63            percentage of features randomly chosen for training
 64
 65        row_sample: float
 66            percentage of rows chosen for training, by stratified bootstrapping
 67
 68        seed: int
 69            reproducibility seed for nodes_sim=='uniform', clustering and dropout
 70
 71        backend: str
 72            "cpu" or "gpu" or "tpu"
 73
 74    Attributes:
 75
 76        beta_: vector
 77            regression coefficients
 78
 79        GCV_: float
 80            Generalized Cross-Validation error
 81
 82    """
 83
 84    # construct the object -----
 85
 86    def __init__(
 87        self,
 88        n_hidden_features=5,
 89        activation_name="relu",
 90        a=0.01,
 91        nodes_sim="sobol",
 92        bias=True,
 93        dropout=0,
 94        direct_link=True,
 95        n_clusters=2,
 96        cluster_encode=True,
 97        type_clust="kmeans",
 98        type_scaling=("std", "std", "std"),
 99        col_sample=1,
100        row_sample=1,
101        seed=123,
102        backend="cpu",
103    ):
104        super().__init__(
105            n_hidden_features=n_hidden_features,
106            activation_name=activation_name,
107            a=a,
108            nodes_sim=nodes_sim,
109            bias=bias,
110            dropout=dropout,
111            direct_link=direct_link,
112            n_clusters=n_clusters,
113            cluster_encode=cluster_encode,
114            type_clust=type_clust,
115            type_scaling=type_scaling,
116            col_sample=col_sample,
117            row_sample=row_sample,
118            seed=seed,
119            backend=backend,
120        )
121
122    def fit(self, X, y, **kwargs):
123        """Fit BaseRegressor to training data (X, y)
124
125        Parameters:
126
127            X: {array-like}, shape = [n_samples, n_features]
128                Training vectors, where n_samples is the number
129                of samples and n_features is the number of features
130
131            y: array-like, shape = [n_samples]
132                Target values
133
134            **kwargs: additional parameters to be passed to self.cook_training_set
135
136        Returns:
137
138            self: object
139        """
140
141        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
142
143        fit_obj = lmf.beta_Sigma_hat(
144            X=scaled_Z, y=centered_y, backend=self.backend
145        )
146
147        self.beta_ = fit_obj["beta_hat"]
148
149        self.GCV_ = fit_obj["GCV"]
150
151        return self
152
153    def predict(self, X, **kwargs):
154        """Predict test data X.
155
156        Parameters:
157
158            X: {array-like}, shape = [n_samples, n_features]
159                Training vectors, where n_samples is the number
160                of samples and n_features is the number of features
161
162            **kwargs: additional parameters to be passed to self.cook_test_set
163
164        Returns:
165
166            model predictions: {array-like}
167        """
168
169        if len(X.shape) == 1:
170            n_features = X.shape[0]
171            new_X = mo.rbind(
172                X.reshape(1, n_features),
173                np.ones(n_features).reshape(1, n_features),
174            )
175
176            return (
177                self.y_mean_
178                + mo.safe_sparse_dot(
179                    a=self.cook_test_set(new_X, **kwargs),
180                    b=self.beta_,
181                    backend=self.backend,
182                )
183            )[0]
184
185        return self.y_mean_ + mo.safe_sparse_dot(
186            a=self.cook_test_set(X, **kwargs),
187            b=self.beta_,
188            backend=self.backend,
189        )

Random Vector Functional Link Network regression without shrinkage

Parameters:

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for hidden layer nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or
    not (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original features are included (True) in model's
    fitting or not (False)

n_clusters: int
    number of clusters for type_clust='kmeans' or type_clust='gmm'
    clustering (could be 0: no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot);
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

col_sample: float
    percentage of features randomly chosen for training

row_sample: float
    percentage of rows chosen for training, by stratified bootstrapping

seed: int
    reproducibility seed for nodes_sim=='uniform', clustering and dropout

backend: str
    "cpu" or "gpu" or "tpu"

Attributes:

beta_: vector
    regression coefficients

GCV_: float
    Generalized Cross-Validation error
def fit(self, X, y, **kwargs):
122    def fit(self, X, y, **kwargs):
123        """Fit BaseRegressor to training data (X, y)
124
125        Parameters:
126
127            X: {array-like}, shape = [n_samples, n_features]
128                Training vectors, where n_samples is the number
129                of samples and n_features is the number of features
130
131            y: array-like, shape = [n_samples]
132                Target values
133
134            **kwargs: additional parameters to be passed to self.cook_training_set
135
136        Returns:
137
138            self: object
139        """
140
141        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
142
143        fit_obj = lmf.beta_Sigma_hat(
144            X=scaled_Z, y=centered_y, backend=self.backend
145        )
146
147        self.beta_ = fit_obj["beta_hat"]
148
149        self.GCV_ = fit_obj["GCV"]
150
151        return self

Fit BaseRegressor to training data (X, y)

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

**kwargs: additional parameters to be passed to self.cook_training_set

Returns:

self: object
def predict(self, X, **kwargs):
153    def predict(self, X, **kwargs):
154        """Predict test data X.
155
156        Parameters:
157
158            X: {array-like}, shape = [n_samples, n_features]
159                Training vectors, where n_samples is the number
160                of samples and n_features is the number of features
161
162            **kwargs: additional parameters to be passed to self.cook_test_set
163
164        Returns:
165
166            model predictions: {array-like}
167        """
168
169        if len(X.shape) == 1:
170            n_features = X.shape[0]
171            new_X = mo.rbind(
172                X.reshape(1, n_features),
173                np.ones(n_features).reshape(1, n_features),
174            )
175
176            return (
177                self.y_mean_
178                + mo.safe_sparse_dot(
179                    a=self.cook_test_set(new_X, **kwargs),
180                    b=self.beta_,
181                    backend=self.backend,
182                )
183            )[0]
184
185        return self.y_mean_ + mo.safe_sparse_dot(
186            a=self.cook_test_set(X, **kwargs),
187            b=self.beta_,
188            backend=self.backend,
189        )

Predict test data X.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features

**kwargs: additional parameters to be passed to self.cook_test_set

Returns:

model predictions: {array-like}
class BayesianRVFLRegressor(nnetsauce.Base, sklearn.base.RegressorMixin):
 15class BayesianRVFLRegressor(Base, RegressorMixin):
 16    """Bayesian Random Vector Functional Link Network regression with one prior
 17
 18    Parameters:
 19
 20        n_hidden_features: int
 21            number of nodes in the hidden layer
 22
 23        activation_name: str
 24            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 25
 26        a: float
 27            hyperparameter for 'prelu' or 'elu' activation function
 28
 29        nodes_sim: str
 30            type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 'uniform'
 31
 32        bias: boolean
 33            indicates if the hidden layer contains a bias term (True) or not (False)
 34
 35        dropout: float
 36            regularization parameter; (random) percentage of nodes dropped out
 37            of the training
 38
 39        direct_link: boolean
 40            indicates if the original features are included (True) in model''s fitting or not (False)
 41
 42        n_clusters: int
 43            number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering)
 44
 45        cluster_encode: bool
 46            defines how the variable containing clusters is treated (default is one-hot)
 47            if `False`, then labels are used, without one-hot encoding
 48
 49        type_clust: str
 50            type of clustering method: currently k-means ('kmeans') or Gaussian Mixture Model ('gmm')
 51
 52        type_scaling: a tuple of 3 strings
 53            scaling methods for inputs, hidden layer, and clustering respectively
 54            (and when relevant).
 55            Currently available: standardization ('std') or MinMax scaling ('minmax')
 56
 57        seed: int
 58            reproducibility seed for nodes_sim=='uniform'
 59
 60        s: float
 61            std. dev. of regression parameters in Bayesian Ridge Regression
 62
 63        sigma: float
 64            std. dev. of residuals in Bayesian Ridge Regression
 65
 66        return_std: boolean
 67            if True, uncertainty around predictions is evaluated
 68
 69        backend: str
 70            "cpu" or "gpu" or "tpu"
 71
 72    Attributes:
 73
 74        beta_: array-like
 75            regression''s coefficients
 76
 77        Sigma_: array-like
 78            covariance of the distribution of fitted parameters
 79
 80        GCV_: float
 81            Generalized cross-validation error
 82
 83        y_mean_: float
 84            average response
 85
 86    Examples:
 87
 88    ```python
 89    TBD
 90    ```
 91
 92    """
 93
 94    # construct the object -----
 95
 96    def __init__(
 97        self,
 98        n_hidden_features=5,
 99        activation_name="relu",
100        a=0.01,
101        nodes_sim="sobol",
102        bias=True,
103        dropout=0,
104        direct_link=True,
105        n_clusters=2,
106        cluster_encode=True,
107        type_clust="kmeans",
108        type_scaling=("std", "std", "std"),
109        seed=123,
110        s=0.1,
111        sigma=0.05,
112        return_std=True,
113        backend="cpu",
114    ):
115        super().__init__(
116            n_hidden_features=n_hidden_features,
117            activation_name=activation_name,
118            a=a,
119            nodes_sim=nodes_sim,
120            bias=bias,
121            dropout=dropout,
122            direct_link=direct_link,
123            n_clusters=n_clusters,
124            cluster_encode=cluster_encode,
125            type_clust=type_clust,
126            type_scaling=type_scaling,
127            seed=seed,
128            backend=backend,
129        )
130        self.s = s
131        self.sigma = sigma
132        self.beta_ = None
133        self.Sigma_ = None
134        self.GCV_ = None
135        self.return_std = return_std
136
137    def fit(self, X, y, **kwargs):
138        """Fit BayesianRVFLRegressor to training data (X, y).
139
140        Parameters:
141
142            X: {array-like}, shape = [n_samples, n_features]
143                Training vectors, where n_samples is the number
144                of samples and n_features is the number of features.
145
146            y: array-like, shape = [n_samples]
147                Target values.
148
149            **kwargs: additional parameters to be passed to
150                    self.cook_training_set
151
152        Returns:
153
154            self: object
155
156        """
157
158        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
159
160        fit_obj = lmf.beta_Sigma_hat_rvfl(
161            X=scaled_Z,
162            y=centered_y,
163            s=self.s,
164            sigma=self.sigma,
165            fit_intercept=False,
166            return_cov=self.return_std,
167            backend=self.backend,
168        )
169
170        self.beta_ = fit_obj["beta_hat"]
171
172        if self.return_std == True:
173            self.Sigma_ = fit_obj["Sigma_hat"]
174
175        self.GCV_ = fit_obj["GCV"]
176
177        return self
178
179    def predict(self, X, return_std=False, **kwargs):
180        """Predict test data X.
181
182        Parameters:
183
184            X: {array-like}, shape = [n_samples, n_features]
185                Training vectors, where n_samples is the number
186                of samples and n_features is the number of features.
187
188            return_std: {boolean}, standard dev. is returned or not
189
190            **kwargs: additional parameters to be passed to
191                    self.cook_test_set
192
193        Returns:
194
195            model predictions: {array-like}
196
197        """
198
199        if len(X.shape) == 1:  # one observation in the test set only
200            n_features = X.shape[0]
201            new_X = mo.rbind(
202                x=X.reshape(1, n_features),
203                y=np.ones(n_features).reshape(1, n_features),
204                backend=self.backend,
205            )
206
207        self.return_std = return_std
208
209        if self.return_std == False:
210            if len(X.shape) == 1:
211                return (
212                    self.y_mean_
213                    + mo.safe_sparse_dot(
214                        a=self.cook_test_set(new_X, **kwargs),
215                        b=self.beta_,
216                        backend=self.backend,
217                    )
218                )[0]
219
220            return self.y_mean_ + mo.safe_sparse_dot(
221                a=self.cook_test_set(X, **kwargs),
222                b=self.beta_,
223                backend=self.backend,
224            )
225
226        else:  # confidence interval required for preds?
227            if len(X.shape) == 1:
228                Z = self.cook_test_set(new_X, **kwargs)
229
230                pred_obj = lmf.beta_Sigma_hat_rvfl(
231                    s=self.s,
232                    sigma=self.sigma,
233                    X_star=Z,
234                    return_cov=True,
235                    beta_hat_=self.beta_,
236                    Sigma_hat_=self.Sigma_,
237                    backend=self.backend,
238                )
239
240                return (
241                    self.y_mean_ + pred_obj["preds"][0],
242                    pred_obj["preds_std"][0],
243                )
244
245            Z = self.cook_test_set(X, **kwargs)
246
247            pred_obj = lmf.beta_Sigma_hat_rvfl(
248                s=self.s,
249                sigma=self.sigma,
250                X_star=Z,
251                return_cov=True,
252                beta_hat_=self.beta_,
253                Sigma_hat_=self.Sigma_,
254                backend=self.backend,
255            )
256
257            return (self.y_mean_ + pred_obj["preds"], pred_obj["preds_std"])

Bayesian Random Vector Functional Link Network regression with one prior

Parameters:

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original features are included (True) in model''s fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

seed: int
    reproducibility seed for nodes_sim=='uniform'

s: float
    std. dev. of regression parameters in Bayesian Ridge Regression

sigma: float
    std. dev. of residuals in Bayesian Ridge Regression

return_std: boolean
    if True, uncertainty around predictions is evaluated

backend: str
    "cpu" or "gpu" or "tpu"

Attributes:

beta_: array-like
    regression''s coefficients

Sigma_: array-like
    covariance of the distribution of fitted parameters

GCV_: float
    Generalized cross-validation error

y_mean_: float
    average response

Examples:

TBD
def fit(self, X, y, **kwargs):
137    def fit(self, X, y, **kwargs):
138        """Fit BayesianRVFLRegressor to training data (X, y).
139
140        Parameters:
141
142            X: {array-like}, shape = [n_samples, n_features]
143                Training vectors, where n_samples is the number
144                of samples and n_features is the number of features.
145
146            y: array-like, shape = [n_samples]
147                Target values.
148
149            **kwargs: additional parameters to be passed to
150                    self.cook_training_set
151
152        Returns:
153
154            self: object
155
156        """
157
158        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
159
160        fit_obj = lmf.beta_Sigma_hat_rvfl(
161            X=scaled_Z,
162            y=centered_y,
163            s=self.s,
164            sigma=self.sigma,
165            fit_intercept=False,
166            return_cov=self.return_std,
167            backend=self.backend,
168        )
169
170        self.beta_ = fit_obj["beta_hat"]
171
172        if self.return_std == True:
173            self.Sigma_ = fit_obj["Sigma_hat"]
174
175        self.GCV_ = fit_obj["GCV"]
176
177        return self

Fit BayesianRVFLRegressor to training data (X, y).

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set

Returns:

self: object
def predict(self, X, return_std=False, **kwargs):
179    def predict(self, X, return_std=False, **kwargs):
180        """Predict test data X.
181
182        Parameters:
183
184            X: {array-like}, shape = [n_samples, n_features]
185                Training vectors, where n_samples is the number
186                of samples and n_features is the number of features.
187
188            return_std: {boolean}, standard dev. is returned or not
189
190            **kwargs: additional parameters to be passed to
191                    self.cook_test_set
192
193        Returns:
194
195            model predictions: {array-like}
196
197        """
198
199        if len(X.shape) == 1:  # one observation in the test set only
200            n_features = X.shape[0]
201            new_X = mo.rbind(
202                x=X.reshape(1, n_features),
203                y=np.ones(n_features).reshape(1, n_features),
204                backend=self.backend,
205            )
206
207        self.return_std = return_std
208
209        if self.return_std == False:
210            if len(X.shape) == 1:
211                return (
212                    self.y_mean_
213                    + mo.safe_sparse_dot(
214                        a=self.cook_test_set(new_X, **kwargs),
215                        b=self.beta_,
216                        backend=self.backend,
217                    )
218                )[0]
219
220            return self.y_mean_ + mo.safe_sparse_dot(
221                a=self.cook_test_set(X, **kwargs),
222                b=self.beta_,
223                backend=self.backend,
224            )
225
226        else:  # confidence interval required for preds?
227            if len(X.shape) == 1:
228                Z = self.cook_test_set(new_X, **kwargs)
229
230                pred_obj = lmf.beta_Sigma_hat_rvfl(
231                    s=self.s,
232                    sigma=self.sigma,
233                    X_star=Z,
234                    return_cov=True,
235                    beta_hat_=self.beta_,
236                    Sigma_hat_=self.Sigma_,
237                    backend=self.backend,
238                )
239
240                return (
241                    self.y_mean_ + pred_obj["preds"][0],
242                    pred_obj["preds_std"][0],
243                )
244
245            Z = self.cook_test_set(X, **kwargs)
246
247            pred_obj = lmf.beta_Sigma_hat_rvfl(
248                s=self.s,
249                sigma=self.sigma,
250                X_star=Z,
251                return_cov=True,
252                beta_hat_=self.beta_,
253                Sigma_hat_=self.Sigma_,
254                backend=self.backend,
255            )
256
257            return (self.y_mean_ + pred_obj["preds"], pred_obj["preds_std"])

Predict test data X.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

return_std: {boolean}, standard dev. is returned or not

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
class BayesianRVFL2Regressor(nnetsauce.Base, sklearn.base.RegressorMixin):
 15class BayesianRVFL2Regressor(Base, RegressorMixin):
 16    """Bayesian Random Vector Functional Link Network regression with two priors
 17
 18    Parameters:
 19
 20        n_hidden_features: int
 21            number of nodes in the hidden layer
 22
 23        activation_name: str
 24            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 25
 26        a: float
 27            hyperparameter for 'prelu' or 'elu' activation function
 28
 29        nodes_sim: str
 30            type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 'uniform'
 31
 32        bias: boolean
 33            indicates if the hidden layer contains a bias term (True) or not (False)
 34
 35        dropout: float
 36            regularization parameter; (random) percentage of nodes dropped out
 37            of the training
 38
 39        direct_link: boolean
 40            indicates if the original features are included (True) in model''s fitting or not (False)
 41
 42        n_clusters: int
 43            number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering)
 44
 45        cluster_encode: bool
 46            defines how the variable containing clusters is treated (default is one-hot)
 47            if `False`, then labels are used, without one-hot encoding
 48
 49        type_clust: str
 50            type of clustering method: currently k-means ('kmeans') or Gaussian Mixture Model ('gmm')
 51
 52        type_scaling: a tuple of 3 strings
 53            scaling methods for inputs, hidden layer, and clustering respectively
 54            (and when relevant).
 55            Currently available: standardization ('std') or MinMax scaling ('minmax')
 56
 57        seed: int
 58            reproducibility seed for nodes_sim=='uniform'
 59
 60        s1: float
 61            std. dev. of init. regression parameters in Bayesian Ridge Regression
 62
 63        s2: float
 64            std. dev. of augmented regression parameters in Bayesian Ridge Regression
 65
 66        sigma: float
 67            std. dev. of residuals in Bayesian Ridge Regression
 68
 69        return_std: boolean
 70            if True, uncertainty around predictions is evaluated
 71
 72        backend: str
 73            "cpu" or "gpu" or "tpu"
 74
 75    Attributes:
 76
 77        beta_: array-like
 78            regression''s coefficients
 79
 80        Sigma_: array-like
 81            covariance of the distribution of fitted parameters
 82
 83        GCV_: float
 84            Generalized cross-validation error
 85
 86        y_mean_: float
 87            average response
 88
 89    Examples:
 90
 91    ```python
 92    TBD
 93    ```
 94
 95    """
 96
 97    # construct the object -----
 98
 99    def __init__(
100        self,
101        n_hidden_features=5,
102        activation_name="relu",
103        a=0.01,
104        nodes_sim="sobol",
105        bias=True,
106        dropout=0,
107        direct_link=True,
108        n_clusters=0,
109        cluster_encode=True,
110        type_clust="kmeans",
111        type_scaling=("std", "std", "std"),
112        seed=123,
113        s1=0.1,
114        s2=0.1,
115        sigma=0.05,
116        return_std=True,
117        backend="cpu",
118    ):
119        super().__init__(
120            n_hidden_features=n_hidden_features,
121            activation_name=activation_name,
122            a=a,
123            nodes_sim=nodes_sim,
124            bias=bias,
125            dropout=dropout,
126            direct_link=direct_link,
127            n_clusters=n_clusters,
128            cluster_encode=cluster_encode,
129            type_clust=type_clust,
130            type_scaling=type_scaling,
131            seed=seed,
132            backend=backend,
133        )
134
135        self.s1 = s1
136        self.s2 = s2
137        self.sigma = sigma
138        self.beta_ = None
139        self.Sigma_ = None
140        self.GCV_ = None
141        self.return_std = return_std
142
143    def fit(self, X, y, **kwargs):
144        """Fit BayesianRVFL2Regressor to training data (X, y)
145
146        Parameters:
147
148            X: {array-like}, shape = [n_samples, n_features]
149                Training vectors, where n_samples is the number
150                of samples and n_features is the number of features
151
152            y: array-like, shape = [n_samples]
153                Target values
154
155            **kwargs: additional parameters to be passed to
156                    self.cook_training_set
157
158        Returns:
159
160            self: object
161
162        """
163
164        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
165
166        n, p = X.shape
167        q = self.n_hidden_features
168
169        if self.direct_link == True:
170            r = p + self.n_clusters
171
172            block11 = (self.s1**2) * np.eye(r)
173            block12 = np.zeros((r, q))
174            block21 = np.zeros((q, r))
175            block22 = (self.s2**2) * np.eye(q)
176
177            Sigma_prior = mo.rbind(
178                x=mo.cbind(x=block11, y=block12, backend=self.backend),
179                y=mo.cbind(x=block21, y=block22, backend=self.backend),
180                backend=self.backend,
181            )
182
183        else:
184            Sigma_prior = (self.s2**2) * np.eye(q)
185
186        fit_obj = lmf.beta_Sigma_hat_rvfl2(
187            X=scaled_Z,
188            y=centered_y,
189            Sigma=Sigma_prior,
190            sigma=self.sigma,
191            fit_intercept=False,
192            return_cov=self.return_std,
193            backend=self.backend,
194        )
195
196        self.beta_ = fit_obj["beta_hat"]
197
198        if self.return_std == True:
199            self.Sigma_ = fit_obj["Sigma_hat"]
200
201        self.GCV_ = fit_obj["GCV"]
202
203        return self
204
205    def predict(self, X, return_std=False, **kwargs):
206        """Predict test data X.
207
208        Parameters:
209
210            X: {array-like}, shape = [n_samples, n_features]
211                Training vectors, where n_samples is the number
212                of samples and n_features is the number of features.
213
214            return_std: {boolean}, standard dev. is returned or not
215
216            **kwargs: additional parameters to be passed to
217                    self.cook_test_set
218
219        Returns:
220
221            model predictions: {array-like}
222
223        """
224
225        if len(X.shape) == 1:  # one observation in the test set only
226            n_features = X.shape[0]
227            new_X = mo.rbind(
228                x=X.reshape(1, n_features),
229                y=np.ones(n_features).reshape(1, n_features),
230                backend=self.backend,
231            )
232
233        self.return_std = return_std
234
235        if self.return_std == False:
236            if len(X.shape) == 1:
237                return (
238                    self.y_mean_
239                    + mo.safe_sparse_dot(
240                        self.cook_test_set(new_X, **kwargs),
241                        self.beta_,
242                        backend=self.backend,
243                    )
244                )[0]
245
246            return self.y_mean_ + mo.safe_sparse_dot(
247                self.cook_test_set(X, **kwargs),
248                self.beta_,
249                backend=self.backend,
250            )
251
252        else:  # confidence interval required for preds?
253            if len(X.shape) == 1:
254                Z = self.cook_test_set(new_X, **kwargs)
255
256                pred_obj = lmf.beta_Sigma_hat_rvfl2(
257                    X_star=Z,
258                    return_cov=self.return_std,
259                    beta_hat_=self.beta_,
260                    Sigma_hat_=self.Sigma_,
261                    backend=self.backend,
262                )
263
264                return (
265                    self.y_mean_ + pred_obj["preds"][0],
266                    pred_obj["preds_std"][0],
267                )
268
269            Z = self.cook_test_set(X, **kwargs)
270
271            pred_obj = lmf.beta_Sigma_hat_rvfl2(
272                X_star=Z,
273                return_cov=self.return_std,
274                beta_hat_=self.beta_,
275                Sigma_hat_=self.Sigma_,
276                backend=self.backend,
277            )
278
279            return (self.y_mean_ + pred_obj["preds"], pred_obj["preds_std"])

Bayesian Random Vector Functional Link Network regression with two priors

Parameters:

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original features are included (True) in model''s fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

seed: int
    reproducibility seed for nodes_sim=='uniform'

s1: float
    std. dev. of init. regression parameters in Bayesian Ridge Regression

s2: float
    std. dev. of augmented regression parameters in Bayesian Ridge Regression

sigma: float
    std. dev. of residuals in Bayesian Ridge Regression

return_std: boolean
    if True, uncertainty around predictions is evaluated

backend: str
    "cpu" or "gpu" or "tpu"

Attributes:

beta_: array-like
    regression''s coefficients

Sigma_: array-like
    covariance of the distribution of fitted parameters

GCV_: float
    Generalized cross-validation error

y_mean_: float
    average response

Examples:

TBD
def fit(self, X, y, **kwargs):
143    def fit(self, X, y, **kwargs):
144        """Fit BayesianRVFL2Regressor to training data (X, y)
145
146        Parameters:
147
148            X: {array-like}, shape = [n_samples, n_features]
149                Training vectors, where n_samples is the number
150                of samples and n_features is the number of features
151
152            y: array-like, shape = [n_samples]
153                Target values
154
155            **kwargs: additional parameters to be passed to
156                    self.cook_training_set
157
158        Returns:
159
160            self: object
161
162        """
163
164        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
165
166        n, p = X.shape
167        q = self.n_hidden_features
168
169        if self.direct_link == True:
170            r = p + self.n_clusters
171
172            block11 = (self.s1**2) * np.eye(r)
173            block12 = np.zeros((r, q))
174            block21 = np.zeros((q, r))
175            block22 = (self.s2**2) * np.eye(q)
176
177            Sigma_prior = mo.rbind(
178                x=mo.cbind(x=block11, y=block12, backend=self.backend),
179                y=mo.cbind(x=block21, y=block22, backend=self.backend),
180                backend=self.backend,
181            )
182
183        else:
184            Sigma_prior = (self.s2**2) * np.eye(q)
185
186        fit_obj = lmf.beta_Sigma_hat_rvfl2(
187            X=scaled_Z,
188            y=centered_y,
189            Sigma=Sigma_prior,
190            sigma=self.sigma,
191            fit_intercept=False,
192            return_cov=self.return_std,
193            backend=self.backend,
194        )
195
196        self.beta_ = fit_obj["beta_hat"]
197
198        if self.return_std == True:
199            self.Sigma_ = fit_obj["Sigma_hat"]
200
201        self.GCV_ = fit_obj["GCV"]
202
203        return self

Fit BayesianRVFL2Regressor to training data (X, y)

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

**kwargs: additional parameters to be passed to
        self.cook_training_set

Returns:

self: object
def predict(self, X, return_std=False, **kwargs):
205    def predict(self, X, return_std=False, **kwargs):
206        """Predict test data X.
207
208        Parameters:
209
210            X: {array-like}, shape = [n_samples, n_features]
211                Training vectors, where n_samples is the number
212                of samples and n_features is the number of features.
213
214            return_std: {boolean}, standard dev. is returned or not
215
216            **kwargs: additional parameters to be passed to
217                    self.cook_test_set
218
219        Returns:
220
221            model predictions: {array-like}
222
223        """
224
225        if len(X.shape) == 1:  # one observation in the test set only
226            n_features = X.shape[0]
227            new_X = mo.rbind(
228                x=X.reshape(1, n_features),
229                y=np.ones(n_features).reshape(1, n_features),
230                backend=self.backend,
231            )
232
233        self.return_std = return_std
234
235        if self.return_std == False:
236            if len(X.shape) == 1:
237                return (
238                    self.y_mean_
239                    + mo.safe_sparse_dot(
240                        self.cook_test_set(new_X, **kwargs),
241                        self.beta_,
242                        backend=self.backend,
243                    )
244                )[0]
245
246            return self.y_mean_ + mo.safe_sparse_dot(
247                self.cook_test_set(X, **kwargs),
248                self.beta_,
249                backend=self.backend,
250            )
251
252        else:  # confidence interval required for preds?
253            if len(X.shape) == 1:
254                Z = self.cook_test_set(new_X, **kwargs)
255
256                pred_obj = lmf.beta_Sigma_hat_rvfl2(
257                    X_star=Z,
258                    return_cov=self.return_std,
259                    beta_hat_=self.beta_,
260                    Sigma_hat_=self.Sigma_,
261                    backend=self.backend,
262                )
263
264                return (
265                    self.y_mean_ + pred_obj["preds"][0],
266                    pred_obj["preds_std"][0],
267                )
268
269            Z = self.cook_test_set(X, **kwargs)
270
271            pred_obj = lmf.beta_Sigma_hat_rvfl2(
272                X_star=Z,
273                return_cov=self.return_std,
274                beta_hat_=self.beta_,
275                Sigma_hat_=self.Sigma_,
276                backend=self.backend,
277            )
278
279            return (self.y_mean_ + pred_obj["preds"], pred_obj["preds_std"])

Predict test data X.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

return_std: {boolean}, standard dev. is returned or not

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
class ClassicalMTS(nnetsauce.Base):
 35class ClassicalMTS(Base):
 36    """Multivariate time series (FactorMTS) forecasting with Factor models
 37
 38    Parameters:
 39
 40        model: type of model: str.
 41            currently, 'VAR' or 'VECM'.
 42
 43    Attributes:
 44
 45        df_: data frame
 46            the input data frame, in case a data.frame is provided to `fit`
 47
 48        level_: int
 49            level of confidence for prediction intervals (default is 95)
 50
 51    Examples:
 52    See examples/classical_mts_timeseries.py
 53    """
 54
 55    # construct the object -----
 56
 57    def __init__(self, model="VAR"):
 58
 59        self.model = model
 60        if self.model == "VAR":
 61            self.obj = VAR
 62        elif self.model == "VECM":
 63            self.obj = VECM
 64        self.n_series = None
 65        self.mean_ = None
 66        self.upper_ = None
 67        self.lower_ = None
 68        self.output_dates_ = None
 69        self.alpha_ = None
 70        self.df_ = None
 71        self.residuals_ = []
 72        self.sims_ = None
 73        self.level_ = None
 74
 75    def fit(self, X, **kwargs):
 76        """Fit FactorMTS model to training data X, with optional regressors xreg
 77
 78        Parameters:
 79
 80        X: {array-like}, shape = [n_samples, n_features]
 81            Training time series, where n_samples is the number
 82            of samples and n_features is the number of features;
 83            X must be in increasing order (most recent observations last)
 84
 85        **kwargs: for now, additional parameters to be passed to for kernel density estimation, when needed (see sklearn.neighbors.KernelDensity)
 86
 87        Returns:
 88
 89        self: object
 90        """
 91
 92        self.n_series = X.shape[1]
 93
 94        if (
 95            isinstance(X, pd.DataFrame) is False
 96        ):  # input data set is a numpy array
 97            X = pd.DataFrame(X)
 98            self.series_names = ["series" + str(i) for i in range(X.shape[1])]
 99
100        else:  # input data set is a DataFrame with column names
101
102            X_index = None
103            if X.index is not None:
104                X_index = X.index
105                X = copy.deepcopy(mo.convert_df_to_numeric(X))
106            if X_index is not None:
107                X.index = X_index
108            self.series_names = X.columns.tolist()
109
110        if isinstance(X, pd.DataFrame):
111            self.df_ = X
112            X = X.values
113            self.df_.columns = self.series_names
114            self.input_dates = ts.compute_input_dates(self.df_)
115        else:
116            self.df_ = pd.DataFrame(X, columns=self.series_names)
117            self.input_dates = ts.compute_input_dates(self.df_)
118
119        self.obj = self.obj(X, **kwargs).fit(**kwargs)
120
121        return self
122
123    def predict(self, h=5, level=95, **kwargs):
124        """Forecast all the time series, h steps ahead
125
126        Parameters:
127
128        h: {integer}
129            Forecasting horizon
130
131        **kwargs: additional parameters to be passed to
132                self.cook_test_set
133
134        Returns:
135
136        model predictions for horizon = h: {array-like}
137
138        """
139
140        self.output_dates_, frequency = ts.compute_output_dates(self.df_, h)
141
142        self.level_ = level
143
144        self.return_std_ = False  # do not remove (/!\)
145
146        self.mean_ = None  # do not remove (/!\)
147
148        self.mean_ = deepcopy(self.y_)  # do not remove (/!\)
149
150        self.lower_ = None  # do not remove (/!\)
151
152        self.upper_ = None  # do not remove (/!\)
153
154        self.sims_ = None  # do not remove (/!\)
155
156        self.level_ = level
157
158        self.alpha_ = 100 - level
159
160        # Named tuple for forecast results
161        DescribeResult = namedtuple(
162            "DescribeResult", ("mean", "lower", "upper")
163        )
164
165        if self.model == "VAR":
166            mean_forecast, lower_bound, upper_bound = (
167                self.obj.forecast_interval(
168                    self.obj.endog, steps=h, alpha=self.alpha_ / 100, **kwargs
169                )
170            )
171
172        elif self.model == "VECM":
173            forecast_result = self.obj.predict(steps=h)
174            mean_forecast = forecast_result
175            lower_bound, upper_bound = self._compute_confidence_intervals(
176                forecast_result, alpha=self.alpha_ / 100, **kwargs
177            )
178
179        self.mean_ = pd.DataFrame(mean_forecast, columns=self.series_names)
180        self.mean_.index = self.output_dates_
181        self.lower_ = pd.DataFrame(lower_bound, columns=self.series_names)
182        self.lower_.index = self.output_dates_
183        self.upper_ = pd.DataFrame(upper_bound, columns=self.series_names)
184        self.upper_.index = self.output_dates_
185
186        return DescribeResult(
187            mean=self.mean_, lower=self.lower_, upper=self.upper_
188        )
189
190    def _compute_confidence_intervals(self, forecast_result, alpha):
191        """
192        Compute confidence intervals for VECM forecasts.
193        Uses the covariance of residuals to approximate the confidence intervals.
194        """
195        residuals = self.obj.resid
196        cov_matrix = np.cov(residuals.T)  # Covariance matrix of residuals
197        std_errors = np.sqrt(np.diag(cov_matrix))  # Standard errors
198
199        z_value = norm.ppf(1 - alpha / 2)  # Z-score for the given alpha level
200        lower_bound = forecast_result - z_value * std_errors
201        upper_bound = forecast_result + z_value * std_errors
202
203        return lower_bound, upper_bound
204
205    def score(self, X, training_index, testing_index, scoring=None, **kwargs):
206        """Train on training_index, score on testing_index."""
207
208        assert (
209            bool(set(training_index).intersection(set(testing_index))) == False
210        ), "Non-overlapping 'training_index' and 'testing_index' required"
211
212        # Dimensions
213        try:
214            # multivariate time series
215            n, p = X.shape
216        except:
217            # univariate time series
218            n = X.shape[0]
219            p = 1
220
221        # Training and testing sets
222        if p > 1:
223            X_train = X[training_index, :]
224            X_test = X[testing_index, :]
225        else:
226            X_train = X[training_index]
227            X_test = X[testing_index]
228
229        # Horizon
230        h = len(testing_index)
231        assert (
232            len(training_index) + h
233        ) <= n, "Please check lengths of training and testing windows"
234
235        # Fit and predict
236        self.fit(X_train, **kwargs)
237        preds = self.predict(h=h, **kwargs)
238
239        if scoring is None:
240            scoring = "neg_root_mean_squared_error"
241
242        # check inputs
243        assert scoring in (
244            "explained_variance",
245            "neg_mean_absolute_error",
246            "neg_mean_squared_error",
247            "neg_root_mean_squared_error",
248            "neg_mean_squared_log_error",
249            "neg_median_absolute_error",
250            "r2",
251        ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
252                               'neg_mean_squared_error', 'neg_root_mean_squared_error', 'neg_mean_squared_log_error', \
253                               'neg_median_absolute_error', 'r2')"
254
255        scoring_options = {
256            "explained_variance": skm2.explained_variance_score,
257            "neg_mean_absolute_error": skm2.mean_absolute_error,
258            "neg_mean_squared_error": skm2.mean_squared_error,
259            "neg_root_mean_squared_error": lambda x, y: np.sqrt(
260                skm2.mean_squared_error(x, y)
261            ),
262            "neg_mean_squared_log_error": skm2.mean_squared_log_error,
263            "neg_median_absolute_error": skm2.median_absolute_error,
264            "r2": skm2.r2_score,
265        }
266
267        # if p > 1:
268        #     return tuple(
269        #         [
270        #             scoring_options[scoring](
271        #                 X_test[:, i], preds[:, i]#, **kwargs
272        #             )
273        #             for i in range(p)
274        #         ]
275        #     )
276        # else:
277        return scoring_options[scoring](X_test, preds)
278
279    def plot(self, series=None, type_axis="dates", type_plot="pi"):
280        """Plot time series forecast
281
282        Parameters:
283
284        series: {integer} or {string}
285            series index or name
286
287        """
288
289        assert all(
290            [
291                self.mean_ is not None,
292                self.lower_ is not None,
293                self.upper_ is not None,
294                self.output_dates_ is not None,
295            ]
296        ), "model forecasting must be obtained first (with predict)"
297
298        if series is None:
299            assert (
300                self.n_series == 1
301            ), "please specify series index or name (n_series > 1)"
302            series = 0
303
304        if isinstance(series, str):
305            assert (
306                series in self.series_names
307            ), f"series {series} doesn't exist in the input dataset"
308            series_idx = self.df_.columns.get_loc(series)
309        else:
310            assert isinstance(series, int) and (
311                0 <= series < self.n_series
312            ), f"check series index (< {self.n_series})"
313            series_idx = series
314
315        y_all = list(self.df_.iloc[:, series_idx]) + list(
316            self.mean_.iloc[:, series_idx]
317        )
318        y_test = list(self.mean_.iloc[:, series_idx])
319        n_points_all = len(y_all)
320        n_points_train = self.df_.shape[0]
321
322        if type_axis == "numeric":
323            x_all = [i for i in range(n_points_all)]
324            x_test = [i for i in range(n_points_train, n_points_all)]
325
326        if type_axis == "dates":  # use dates
327            x_all = np.concatenate(
328                (self.input_dates.values, self.output_dates_.values), axis=None
329            )
330            x_test = self.output_dates_.values
331
332        if type_plot == "pi":
333            fig, ax = plt.subplots()
334            ax.plot(x_all, y_all, "-")
335            ax.plot(x_test, y_test, "-", color="orange")
336            ax.fill_between(
337                x_test,
338                self.lower_.iloc[:, series_idx],
339                self.upper_.iloc[:, series_idx],
340                alpha=0.2,
341                color="orange",
342            )
343            if self.replications is None:
344                if self.n_series > 1:
345                    plt.title(
346                        f"prediction intervals for {series}",
347                        loc="left",
348                        fontsize=12,
349                        fontweight=0,
350                        color="black",
351                    )
352                else:
353                    plt.title(
354                        f"prediction intervals for input time series",
355                        loc="left",
356                        fontsize=12,
357                        fontweight=0,
358                        color="black",
359                    )
360                plt.show()
361            else:  # self.replications is not None
362                if self.n_series > 1:
363                    plt.title(
364                        f"prediction intervals for {self.replications} simulations of {series}",
365                        loc="left",
366                        fontsize=12,
367                        fontweight=0,
368                        color="black",
369                    )
370                else:
371                    plt.title(
372                        f"prediction intervals for {self.replications} simulations of input time series",
373                        loc="left",
374                        fontsize=12,
375                        fontweight=0,
376                        color="black",
377                    )
378                plt.show()
379
380        if type_plot == "spaghetti":
381            palette = plt.get_cmap("Set1")
382            sims_ix = getsims(self.sims_, series_idx)
383            plt.plot(x_all, y_all, "-")
384            for col_ix in range(
385                sims_ix.shape[1]
386            ):  # avoid this when there are thousands of simulations
387                plt.plot(
388                    x_test,
389                    sims_ix[:, col_ix],
390                    "-",
391                    color=palette(col_ix),
392                    linewidth=1,
393                    alpha=0.9,
394                )
395            plt.plot(x_all, y_all, "-", color="black")
396            plt.plot(x_test, y_test, "-", color="blue")
397            # Add titles
398            if self.n_series > 1:
399                plt.title(
400                    f"{self.replications} simulations of {series}",
401                    loc="left",
402                    fontsize=12,
403                    fontweight=0,
404                    color="black",
405                )
406            else:
407                plt.title(
408                    f"{self.replications} simulations of input time series",
409                    loc="left",
410                    fontsize=12,
411                    fontweight=0,
412                    color="black",
413                )
414            plt.xlabel("Time")
415            plt.ylabel("Values")
416            # Show the graph
417            plt.show()
418
419    def cross_val_score(
420        self,
421        X,
422        scoring="root_mean_squared_error",
423        n_jobs=None,
424        verbose=0,
425        xreg=None,
426        initial_window=5,
427        horizon=3,
428        fixed_window=False,
429        show_progress=True,
430        level=95,
431        **kwargs,
432    ):
433        """Evaluate a score by time series cross-validation.
434
435        Parameters:
436
437            X: {array-like, sparse matrix} of shape (n_samples, n_features)
438                The data to fit.
439
440            scoring: str or a function
441                A str in ('root_mean_squared_error', 'mean_squared_error', 'mean_error',
442                'mean_absolute_error', 'mean_error', 'mean_percentage_error',
443                'mean_absolute_percentage_error',  'winkler_score', 'coverage')
444                Or a function defined as 'coverage' and 'winkler_score' in `utils.timeseries`
445
446            n_jobs: int, default=None
447                Number of jobs to run in parallel.
448
449            verbose: int, default=0
450                The verbosity level.
451
452            xreg: array-like, optional (default=None)
453                Additional (external) regressors to be passed to `fit`
454                xreg must be in 'increasing' order (most recent observations last)
455
456            initial_window: int
457                initial number of consecutive values in each training set sample
458
459            horizon: int
460                number of consecutive values in test set sample
461
462            fixed_window: boolean
463                if False, all training samples start at index 0, and the training
464                window's size is increasing.
465                if True, the training window's size is fixed, and the window is
466                rolling forward
467
468            show_progress: boolean
469                if True, a progress bar is printed
470
471            **kwargs: dict
472                additional parameters to be passed to `fit` and `predict`
473
474        Returns:
475
476            A tuple: descriptive statistics or errors and raw errors
477
478        """
479        tscv = TimeSeriesSplit()
480
481        tscv_obj = tscv.split(
482            X,
483            initial_window=initial_window,
484            horizon=horizon,
485            fixed_window=fixed_window,
486        )
487
488        if isinstance(scoring, str):
489
490            assert scoring in (
491                "root_mean_squared_error",
492                "mean_squared_error",
493                "mean_error",
494                "mean_absolute_error",
495                "mean_percentage_error",
496                "mean_absolute_percentage_error",
497                "winkler_score",
498                "coverage",
499            ), "must have scoring in ('root_mean_squared_error', 'mean_squared_error', 'mean_error', 'mean_absolute_error', 'mean_error', 'mean_percentage_error', 'mean_absolute_percentage_error',  'winkler_score', 'coverage')"
500
501            def err_func(X_test, X_pred, scoring):
502                if (self.replications is not None) or (
503                    self.type_pi == "gaussian"
504                ):  # probabilistic
505                    if scoring == "winkler_score":
506                        return winkler_score(X_pred, X_test, level=level)
507                    elif scoring == "coverage":
508                        return coverage(X_pred, X_test, level=level)
509                    else:
510                        return mean_errors(
511                            pred=X_pred.mean, actual=X_test, scoring=scoring
512                        )
513                else:  # not probabilistic
514                    return mean_errors(
515                        pred=X_pred, actual=X_test, scoring=scoring
516                    )
517
518        else:  # isinstance(scoring, str) = False
519
520            err_func = scoring
521
522        errors = []
523
524        train_indices = []
525
526        test_indices = []
527
528        for train_index, test_index in tscv_obj:
529            train_indices.append(train_index)
530            test_indices.append(test_index)
531
532        if show_progress is True:
533            iterator = tqdm(
534                zip(train_indices, test_indices), total=len(train_indices)
535            )
536        else:
537            iterator = zip(train_indices, test_indices)
538
539        for train_index, test_index in iterator:
540
541            if verbose == 1:
542                print(f"TRAIN: {train_index}")
543                print(f"TEST: {test_index}")
544
545            if isinstance(X, pd.DataFrame):
546                self.fit(X.iloc[train_index, :], xreg=xreg, **kwargs)
547                X_test = X.iloc[test_index, :]
548            else:
549                self.fit(X[train_index, :], xreg=xreg, **kwargs)
550                X_test = X[test_index, :]
551            X_pred = self.predict(h=int(len(test_index)), level=level, **kwargs)
552
553            errors.append(err_func(X_test, X_pred, scoring))
554
555        res = np.asarray(errors)
556
557        return res, describe(res)

Multivariate time series (FactorMTS) forecasting with Factor models

Parameters:

model: type of model: str.
    currently, 'VAR' or 'VECM'.

Attributes:

df_: data frame
    the input data frame, in case a data.frame is provided to `fit`

level_: int
    level of confidence for prediction intervals (default is 95)

Examples: See examples/classical_mts_timeseries.py

def fit(self, X, **kwargs):
 75    def fit(self, X, **kwargs):
 76        """Fit FactorMTS model to training data X, with optional regressors xreg
 77
 78        Parameters:
 79
 80        X: {array-like}, shape = [n_samples, n_features]
 81            Training time series, where n_samples is the number
 82            of samples and n_features is the number of features;
 83            X must be in increasing order (most recent observations last)
 84
 85        **kwargs: for now, additional parameters to be passed to for kernel density estimation, when needed (see sklearn.neighbors.KernelDensity)
 86
 87        Returns:
 88
 89        self: object
 90        """
 91
 92        self.n_series = X.shape[1]
 93
 94        if (
 95            isinstance(X, pd.DataFrame) is False
 96        ):  # input data set is a numpy array
 97            X = pd.DataFrame(X)
 98            self.series_names = ["series" + str(i) for i in range(X.shape[1])]
 99
100        else:  # input data set is a DataFrame with column names
101
102            X_index = None
103            if X.index is not None:
104                X_index = X.index
105                X = copy.deepcopy(mo.convert_df_to_numeric(X))
106            if X_index is not None:
107                X.index = X_index
108            self.series_names = X.columns.tolist()
109
110        if isinstance(X, pd.DataFrame):
111            self.df_ = X
112            X = X.values
113            self.df_.columns = self.series_names
114            self.input_dates = ts.compute_input_dates(self.df_)
115        else:
116            self.df_ = pd.DataFrame(X, columns=self.series_names)
117            self.input_dates = ts.compute_input_dates(self.df_)
118
119        self.obj = self.obj(X, **kwargs).fit(**kwargs)
120
121        return self

Fit FactorMTS model to training data X, with optional regressors xreg

Parameters:

X: {array-like}, shape = [n_samples, n_features] Training time series, where n_samples is the number of samples and n_features is the number of features; X must be in increasing order (most recent observations last)

**kwargs: for now, additional parameters to be passed to for kernel density estimation, when needed (see sklearn.neighbors.KernelDensity)

Returns:

self: object

def predict(self, h=5, level=95, **kwargs):
123    def predict(self, h=5, level=95, **kwargs):
124        """Forecast all the time series, h steps ahead
125
126        Parameters:
127
128        h: {integer}
129            Forecasting horizon
130
131        **kwargs: additional parameters to be passed to
132                self.cook_test_set
133
134        Returns:
135
136        model predictions for horizon = h: {array-like}
137
138        """
139
140        self.output_dates_, frequency = ts.compute_output_dates(self.df_, h)
141
142        self.level_ = level
143
144        self.return_std_ = False  # do not remove (/!\)
145
146        self.mean_ = None  # do not remove (/!\)
147
148        self.mean_ = deepcopy(self.y_)  # do not remove (/!\)
149
150        self.lower_ = None  # do not remove (/!\)
151
152        self.upper_ = None  # do not remove (/!\)
153
154        self.sims_ = None  # do not remove (/!\)
155
156        self.level_ = level
157
158        self.alpha_ = 100 - level
159
160        # Named tuple for forecast results
161        DescribeResult = namedtuple(
162            "DescribeResult", ("mean", "lower", "upper")
163        )
164
165        if self.model == "VAR":
166            mean_forecast, lower_bound, upper_bound = (
167                self.obj.forecast_interval(
168                    self.obj.endog, steps=h, alpha=self.alpha_ / 100, **kwargs
169                )
170            )
171
172        elif self.model == "VECM":
173            forecast_result = self.obj.predict(steps=h)
174            mean_forecast = forecast_result
175            lower_bound, upper_bound = self._compute_confidence_intervals(
176                forecast_result, alpha=self.alpha_ / 100, **kwargs
177            )
178
179        self.mean_ = pd.DataFrame(mean_forecast, columns=self.series_names)
180        self.mean_.index = self.output_dates_
181        self.lower_ = pd.DataFrame(lower_bound, columns=self.series_names)
182        self.lower_.index = self.output_dates_
183        self.upper_ = pd.DataFrame(upper_bound, columns=self.series_names)
184        self.upper_.index = self.output_dates_
185
186        return DescribeResult(
187            mean=self.mean_, lower=self.lower_, upper=self.upper_
188        )

Forecast all the time series, h steps ahead

Parameters:

h: {integer} Forecasting horizon

**kwargs: additional parameters to be passed to self.cook_test_set

Returns:

model predictions for horizon = h: {array-like}

def score(self, X, training_index, testing_index, scoring=None, **kwargs):
205    def score(self, X, training_index, testing_index, scoring=None, **kwargs):
206        """Train on training_index, score on testing_index."""
207
208        assert (
209            bool(set(training_index).intersection(set(testing_index))) == False
210        ), "Non-overlapping 'training_index' and 'testing_index' required"
211
212        # Dimensions
213        try:
214            # multivariate time series
215            n, p = X.shape
216        except:
217            # univariate time series
218            n = X.shape[0]
219            p = 1
220
221        # Training and testing sets
222        if p > 1:
223            X_train = X[training_index, :]
224            X_test = X[testing_index, :]
225        else:
226            X_train = X[training_index]
227            X_test = X[testing_index]
228
229        # Horizon
230        h = len(testing_index)
231        assert (
232            len(training_index) + h
233        ) <= n, "Please check lengths of training and testing windows"
234
235        # Fit and predict
236        self.fit(X_train, **kwargs)
237        preds = self.predict(h=h, **kwargs)
238
239        if scoring is None:
240            scoring = "neg_root_mean_squared_error"
241
242        # check inputs
243        assert scoring in (
244            "explained_variance",
245            "neg_mean_absolute_error",
246            "neg_mean_squared_error",
247            "neg_root_mean_squared_error",
248            "neg_mean_squared_log_error",
249            "neg_median_absolute_error",
250            "r2",
251        ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
252                               'neg_mean_squared_error', 'neg_root_mean_squared_error', 'neg_mean_squared_log_error', \
253                               'neg_median_absolute_error', 'r2')"
254
255        scoring_options = {
256            "explained_variance": skm2.explained_variance_score,
257            "neg_mean_absolute_error": skm2.mean_absolute_error,
258            "neg_mean_squared_error": skm2.mean_squared_error,
259            "neg_root_mean_squared_error": lambda x, y: np.sqrt(
260                skm2.mean_squared_error(x, y)
261            ),
262            "neg_mean_squared_log_error": skm2.mean_squared_log_error,
263            "neg_median_absolute_error": skm2.median_absolute_error,
264            "r2": skm2.r2_score,
265        }
266
267        # if p > 1:
268        #     return tuple(
269        #         [
270        #             scoring_options[scoring](
271        #                 X_test[:, i], preds[:, i]#, **kwargs
272        #             )
273        #             for i in range(p)
274        #         ]
275        #     )
276        # else:
277        return scoring_options[scoring](X_test, preds)

Train on training_index, score on testing_index.

class CustomClassifier(nnetsauce.custom.custom.Custom, sklearn.base.ClassifierMixin):
 14class CustomClassifier(Custom, ClassifierMixin):
 15    """Custom Classification model
 16
 17    Attributes:
 18
 19        obj: object
 20            any object containing a method fit (obj.fit()) and a method predict
 21            (obj.predict())
 22
 23        n_hidden_features: int
 24            number of nodes in the hidden layer
 25
 26        activation_name: str
 27            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 28
 29        a: float
 30            hyperparameter for 'prelu' or 'elu' activation function
 31
 32        nodes_sim: str
 33            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 34            'uniform'
 35
 36        bias: boolean
 37            indicates if the hidden layer contains a bias term (True) or not
 38            (False)
 39
 40        dropout: float
 41            regularization parameter; (random) percentage of nodes dropped out
 42            of the training
 43
 44        direct_link: boolean
 45            indicates if the original predictors are included (True) in model''s
 46            fitting or not (False)
 47
 48        n_clusters: int
 49            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 50                no clustering)
 51
 52        cluster_encode: bool
 53            defines how the variable containing clusters is treated (default is one-hot)
 54            if `False`, then labels are used, without one-hot encoding
 55
 56        type_clust: str
 57            type of clustering method: currently k-means ('kmeans') or Gaussian
 58            Mixture Model ('gmm')
 59
 60        type_scaling: a tuple of 3 strings
 61            scaling methods for inputs, hidden layer, and clustering respectively
 62            (and when relevant).
 63            Currently available: standardization ('std') or MinMax scaling ('minmax')
 64
 65        col_sample: float
 66            percentage of covariates randomly chosen for training
 67
 68        row_sample: float
 69            percentage of rows chosen for training, by stratified bootstrapping
 70
 71        level: float
 72            confidence level for prediction sets. Default is None.
 73
 74        pi_method: str
 75            method for constructing the prediction sets: 'icp', 'tcp' if level is not None. Default is 'icp'.
 76
 77        seed: int
 78            reproducibility seed for nodes_sim=='uniform'
 79
 80        backend: str
 81            "cpu" or "gpu" or "tpu"
 82
 83    Examples:
 84
 85    Note: it's better to use the `DeepClassifier` or `LazyDeepClassifier` classes directly
 86
 87    ```python
 88    import nnetsauce as ns
 89    from sklearn.ensemble import RandomForestClassifier
 90    from sklearn.model_selection import train_test_split
 91    from sklearn.datasets import load_digits
 92    from time import time
 93
 94    digits = load_digits()
 95    X = digits.data
 96    y = digits.target
 97    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,
 98                                                        random_state=123)
 99
100    # layer 1 (base layer) ----
101    layer1_regr = RandomForestClassifier(n_estimators=10, random_state=123)
102
103    start = time()
104
105    layer1_regr.fit(X_train, y_train)
106
107    # Accuracy in layer 1
108    print(layer1_regr.score(X_test, y_test))
109
110    # layer 2 using layer 1 ----
111    layer2_regr = ns.CustomClassifier(obj = layer1_regr, n_hidden_features=5,
112                            direct_link=True, bias=True,
113                            nodes_sim='uniform', activation_name='relu',
114                            n_clusters=2, seed=123)
115    layer2_regr.fit(X_train, y_train)
116
117    # Accuracy in layer 2
118    print(layer2_regr.score(X_test, y_test))
119
120    # layer 3 using layer 2 ----
121    layer3_regr = ns.CustomClassifier(obj = layer2_regr, n_hidden_features=10,
122                            direct_link=True, bias=True, dropout=0.7,
123                            nodes_sim='uniform', activation_name='relu',
124                            n_clusters=2, seed=123)
125    layer3_regr.fit(X_train, y_train)
126
127    # Accuracy in layer 3
128    print(layer3_regr.score(X_test, y_test))
129
130    print(f"Elapsed {time() - start}")
131    ```
132
133    """
134
135    # construct the object -----
136
137    def __init__(
138        self,
139        obj,
140        n_hidden_features=5,
141        activation_name="relu",
142        a=0.01,
143        nodes_sim="sobol",
144        bias=True,
145        dropout=0,
146        direct_link=True,
147        n_clusters=2,
148        cluster_encode=True,
149        type_clust="kmeans",
150        type_scaling=("std", "std", "std"),
151        col_sample=1,
152        row_sample=1,
153        level=None,
154        pi_method="icp",
155        seed=123,
156        backend="cpu",
157    ):
158        super().__init__(
159            obj=obj,
160            n_hidden_features=n_hidden_features,
161            activation_name=activation_name,
162            a=a,
163            nodes_sim=nodes_sim,
164            bias=bias,
165            dropout=dropout,
166            direct_link=direct_link,
167            n_clusters=n_clusters,
168            cluster_encode=cluster_encode,
169            type_clust=type_clust,
170            type_scaling=type_scaling,
171            col_sample=col_sample,
172            row_sample=row_sample,
173            seed=seed,
174            backend=backend,
175        )
176        self.level = level
177        self.pi_method = pi_method
178        self.type_fit = "classification"
179        if self.level is not None:
180            self.obj = PredictionSet(
181                self.obj, level=self.level, method=self.pi_method
182            )
183
184    def fit(self, X, y, sample_weight=None, **kwargs):
185        """Fit custom model to training data (X, y).
186
187        Parameters:
188
189            X: {array-like}, shape = [n_samples, n_features]
190                Training vectors, where n_samples is the number
191                of samples and n_features is the number of features.
192
193            y: array-like, shape = [n_samples]
194                Target values.
195
196            **kwargs: additional parameters to be passed to
197                        self.cook_training_set or self.obj.fit
198
199        Returns:
200
201            self: object
202        """
203
204        output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
205        self.n_classes_ = len(np.unique(y))  # for compatibility with sklearn
206
207        # if sample_weights, else: (must use self.row_index)
208        if sample_weight is not None:
209            self.obj.fit(
210                scaled_Z,
211                output_y,
212                sample_weight=np.ravel(sample_weight, order="C")[
213                    self.index_row_
214                ],
215                # **kwargs
216            )
217
218            return self
219
220        # if sample_weight is None:
221        self.obj.fit(scaled_Z, output_y)
222        self.classes_ = np.unique(y)  # for compatibility with sklearn
223        self.n_classes_ = len(self.classes_)  # for compatibility with sklearn
224
225        return self
226
227    def partial_fit(self, X, y, sample_weight=None, **kwargs):
228        """Partial fit custom model to training data (X, y).
229
230        Parameters:
231
232            X: {array-like}, shape = [n_samples, n_features]
233                Subset of training vectors, where n_samples is the number
234                of samples and n_features is the number of features.
235
236            y: array-like, shape = [n_samples]
237                Subset of target values.
238
239            **kwargs: additional parameters to be passed to
240                        self.cook_training_set or self.obj.fit
241
242        Returns:
243
244            self: object
245        """
246
247        output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
248        self.n_classes_ = len(np.unique(y))  # for compatibility with sklearn
249
250        # if sample_weights, else: (must use self.row_index)
251        if sample_weight is not None:
252            try:
253                self.obj.partial_fit(
254                    scaled_Z,
255                    output_y,
256                    sample_weight=np.ravel(sample_weight, order="C")[
257                        self.index_row_
258                    ],
259                    # **kwargs
260                )
261            except:
262                NotImplementedError
263
264            return self
265
266        # if sample_weight is None:
267        try:
268            self.obj.fit(scaled_Z, output_y)
269        except:
270            raise NotImplementedError
271
272        self.classes_ = np.unique(y)  # for compatibility with sklearn
273        self.n_classes_ = len(self.classes_)  # for compatibility with sklearn
274
275        return self
276
277    def predict(self, X, **kwargs):
278        """Predict test data X.
279
280        Parameters:
281
282            X: {array-like}, shape = [n_samples, n_features]
283                Training vectors, where n_samples is the number
284                of samples and n_features is the number of features.
285
286            **kwargs: additional parameters to be passed to
287                    self.cook_test_set
288
289        Returns:
290
291            model predictions: {array-like}
292        """
293
294        if len(X.shape) == 1:
295            n_features = X.shape[0]
296            new_X = mo.rbind(
297                X.reshape(1, n_features),
298                np.ones(n_features).reshape(1, n_features),
299            )
300
301            return (
302                self.obj.predict(self.cook_test_set(new_X, **kwargs), **kwargs)
303            )[0]
304
305        return self.obj.predict(self.cook_test_set(X, **kwargs), **kwargs)
306
307    def predict_proba(self, X, **kwargs):
308        """Predict probabilities for test data X.
309
310        Args:
311
312            X: {array-like}, shape = [n_samples, n_features]
313                Training vectors, where n_samples is the number
314                of samples and n_features is the number of features.
315
316            **kwargs: additional parameters to be passed to
317                    self.cook_test_set
318
319        Returns:
320
321            probability estimates for test data: {array-like}
322        """
323
324        if len(X.shape) == 1:
325            n_features = X.shape[0]
326            new_X = mo.rbind(
327                X.reshape(1, n_features),
328                np.ones(n_features).reshape(1, n_features),
329            )
330
331            return (
332                self.obj.predict_proba(
333                    self.cook_test_set(new_X, **kwargs), **kwargs
334                )
335            )[0]
336
337        return self.obj.predict_proba(self.cook_test_set(X, **kwargs), **kwargs)

Custom Classification model

Attributes:

obj: object
    any object containing a method fit (obj.fit()) and a method predict
    (obj.predict())

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original predictors are included (True) in model''s
    fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

col_sample: float
    percentage of covariates randomly chosen for training

row_sample: float
    percentage of rows chosen for training, by stratified bootstrapping

level: float
    confidence level for prediction sets. Default is None.

pi_method: str
    method for constructing the prediction sets: 'icp', 'tcp' if level is not None. Default is 'icp'.

seed: int
    reproducibility seed for nodes_sim=='uniform'

backend: str
    "cpu" or "gpu" or "tpu"

Examples:

Note: it's better to use the DeepClassifier or LazyDeepClassifier classes directly

import nnetsauce as ns
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_digits
from time import time

digits = load_digits()
X = digits.data
y = digits.target
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,
                                                    random_state=123)

# layer 1 (base layer) ----
layer1_regr = RandomForestClassifier(n_estimators=10, random_state=123)

start = time()

layer1_regr.fit(X_train, y_train)

# Accuracy in layer 1
print(layer1_regr.score(X_test, y_test))

# layer 2 using layer 1 ----
layer2_regr = ns.CustomClassifier(obj = layer1_regr, n_hidden_features=5,
                        direct_link=True, bias=True,
                        nodes_sim='uniform', activation_name='relu',
                        n_clusters=2, seed=123)
layer2_regr.fit(X_train, y_train)

# Accuracy in layer 2
print(layer2_regr.score(X_test, y_test))

# layer 3 using layer 2 ----
layer3_regr = ns.CustomClassifier(obj = layer2_regr, n_hidden_features=10,
                        direct_link=True, bias=True, dropout=0.7,
                        nodes_sim='uniform', activation_name='relu',
                        n_clusters=2, seed=123)
layer3_regr.fit(X_train, y_train)

# Accuracy in layer 3
print(layer3_regr.score(X_test, y_test))

print(f"Elapsed {time() - start}")
def fit(self, X, y, sample_weight=None, **kwargs):
184    def fit(self, X, y, sample_weight=None, **kwargs):
185        """Fit custom model to training data (X, y).
186
187        Parameters:
188
189            X: {array-like}, shape = [n_samples, n_features]
190                Training vectors, where n_samples is the number
191                of samples and n_features is the number of features.
192
193            y: array-like, shape = [n_samples]
194                Target values.
195
196            **kwargs: additional parameters to be passed to
197                        self.cook_training_set or self.obj.fit
198
199        Returns:
200
201            self: object
202        """
203
204        output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
205        self.n_classes_ = len(np.unique(y))  # for compatibility with sklearn
206
207        # if sample_weights, else: (must use self.row_index)
208        if sample_weight is not None:
209            self.obj.fit(
210                scaled_Z,
211                output_y,
212                sample_weight=np.ravel(sample_weight, order="C")[
213                    self.index_row_
214                ],
215                # **kwargs
216            )
217
218            return self
219
220        # if sample_weight is None:
221        self.obj.fit(scaled_Z, output_y)
222        self.classes_ = np.unique(y)  # for compatibility with sklearn
223        self.n_classes_ = len(self.classes_)  # for compatibility with sklearn
224
225        return self

Fit custom model to training data (X, y).

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
            self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, **kwargs):
277    def predict(self, X, **kwargs):
278        """Predict test data X.
279
280        Parameters:
281
282            X: {array-like}, shape = [n_samples, n_features]
283                Training vectors, where n_samples is the number
284                of samples and n_features is the number of features.
285
286            **kwargs: additional parameters to be passed to
287                    self.cook_test_set
288
289        Returns:
290
291            model predictions: {array-like}
292        """
293
294        if len(X.shape) == 1:
295            n_features = X.shape[0]
296            new_X = mo.rbind(
297                X.reshape(1, n_features),
298                np.ones(n_features).reshape(1, n_features),
299            )
300
301            return (
302                self.obj.predict(self.cook_test_set(new_X, **kwargs), **kwargs)
303            )[0]
304
305        return self.obj.predict(self.cook_test_set(X, **kwargs), **kwargs)

Predict test data X.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
def predict_proba(self, X, **kwargs):
307    def predict_proba(self, X, **kwargs):
308        """Predict probabilities for test data X.
309
310        Args:
311
312            X: {array-like}, shape = [n_samples, n_features]
313                Training vectors, where n_samples is the number
314                of samples and n_features is the number of features.
315
316            **kwargs: additional parameters to be passed to
317                    self.cook_test_set
318
319        Returns:
320
321            probability estimates for test data: {array-like}
322        """
323
324        if len(X.shape) == 1:
325            n_features = X.shape[0]
326            new_X = mo.rbind(
327                X.reshape(1, n_features),
328                np.ones(n_features).reshape(1, n_features),
329            )
330
331            return (
332                self.obj.predict_proba(
333                    self.cook_test_set(new_X, **kwargs), **kwargs
334                )
335            )[0]
336
337        return self.obj.predict_proba(self.cook_test_set(X, **kwargs), **kwargs)

Predict probabilities for test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

probability estimates for test data: {array-like}
class CustomRegressor(nnetsauce.custom.custom.Custom, sklearn.base.RegressorMixin):
 16class CustomRegressor(Custom, RegressorMixin):
 17    """Custom Regression model
 18
 19    This class is used to 'augment' any regression model with transformed features.
 20
 21    Parameters:
 22
 23        obj: object
 24            any object containing a method fit (obj.fit()) and a method predict
 25            (obj.predict())
 26
 27        n_hidden_features: int
 28            number of nodes in the hidden layer
 29
 30        activation_name: str
 31            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 32
 33        a: float
 34            hyperparameter for 'prelu' or 'elu' activation function
 35
 36        nodes_sim: str
 37            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 38            'uniform'
 39
 40        bias: boolean
 41            indicates if the hidden layer contains a bias term (True) or not
 42            (False)
 43
 44        dropout: float
 45            regularization parameter; (random) percentage of nodes dropped out
 46            of the training
 47
 48        direct_link: boolean
 49            indicates if the original predictors are included (True) in model's
 50            fitting or not (False)
 51
 52        n_clusters: int
 53            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 54                no clustering)
 55
 56        cluster_encode: bool
 57            defines how the variable containing clusters is treated (default is one-hot)
 58            if `False`, then labels are used, without one-hot encoding
 59
 60        type_clust: str
 61            type of clustering method: currently k-means ('kmeans') or Gaussian
 62            Mixture Model ('gmm')
 63
 64        type_scaling: a tuple of 3 strings
 65            scaling methods for inputs, hidden layer, and clustering respectively
 66            (and when relevant).
 67            Currently available: standardization ('std') or MinMax scaling ('minmax')
 68
 69        type_pi: str.
 70            type of prediction interval; currently "kde" (default).
 71            Used only in `self.predict`, for `self.replications` > 0 and `self.kernel`
 72            in ('gaussian', 'tophat'). Default is `None`.
 73
 74        replications: int.
 75            number of replications (if needed) for predictive simulation.
 76            Used only in `self.predict`, for `self.kernel` in ('gaussian',
 77            'tophat') and `self.type_pi = 'kde'`. Default is `None`.
 78
 79        kernel: str.
 80            the kernel to use for kernel density estimation (used for predictive
 81            simulation in `self.predict`, with `method='splitconformal'` and
 82            `type_pi = 'kde'`). Currently, either 'gaussian' or 'tophat'.
 83
 84        type_split: str.
 85            Type of splitting for conformal prediction. None (default), or
 86            "random" (random split of data) or "sequential" (sequential split of data)
 87
 88        col_sample: float
 89            percentage of covariates randomly chosen for training
 90
 91        row_sample: float
 92            percentage of rows chosen for training, by stratified bootstrapping
 93
 94        level: float
 95            confidence level for prediction intervals
 96
 97        pi_method: str
 98            method for prediction intervals: 'splitconformal' or 'localconformal'
 99
100        seed: int
101            reproducibility seed for nodes_sim=='uniform'
102
103        type_fit: str
104            'regression'
105
106        backend: str
107            "cpu" or "gpu" or "tpu"
108
109    Examples:
110
111    See [https://thierrymoudiki.github.io/blog/2024/03/18/python/conformal-and-bayesian-regression](https://thierrymoudiki.github.io/blog/2024/03/18/python/conformal-and-bayesian-regression)
112
113    """
114
115    # construct the object -----
116
117    def __init__(
118        self,
119        obj,
120        n_hidden_features=5,
121        activation_name="relu",
122        a=0.01,
123        nodes_sim="sobol",
124        bias=True,
125        dropout=0,
126        direct_link=True,
127        n_clusters=2,
128        cluster_encode=True,
129        type_clust="kmeans",
130        type_scaling=("std", "std", "std"),
131        type_pi=None,
132        replications=None,
133        kernel=None,
134        type_split=None,
135        col_sample=1,
136        row_sample=1,
137        level=None,
138        pi_method=None,
139        seed=123,
140        backend="cpu",
141    ):
142        super().__init__(
143            obj=obj,
144            n_hidden_features=n_hidden_features,
145            activation_name=activation_name,
146            a=a,
147            nodes_sim=nodes_sim,
148            bias=bias,
149            dropout=dropout,
150            direct_link=direct_link,
151            n_clusters=n_clusters,
152            cluster_encode=cluster_encode,
153            type_clust=type_clust,
154            type_scaling=type_scaling,
155            col_sample=col_sample,
156            row_sample=row_sample,
157            seed=seed,
158            backend=backend,
159        )
160
161        self.type_fit = "regression"
162        self.type_pi = type_pi
163        self.replications = replications
164        self.kernel = kernel
165        self.type_split = type_split
166        self.level = level
167        self.pi_method = pi_method
168
169    def fit(self, X, y, sample_weight=None, **kwargs):
170        """Fit custom model to training data (X, y).
171
172        Parameters:
173
174            X: {array-like}, shape = [n_samples, n_features]
175                Training vectors, where n_samples is the number
176                of samples and n_features is the number of features.
177
178            y: array-like, shape = [n_samples]
179                Target values.
180
181            **kwargs: additional parameters to be passed to
182                self.cook_training_set or self.obj.fit
183
184        Returns:
185
186            self: object
187
188        """
189
190        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
191
192        # if sample_weights, else: (must use self.row_index)
193        if sample_weight is not None:
194            self.obj.fit(
195                scaled_Z,
196                centered_y,
197                sample_weight=np.ravel(sample_weight, order="C")[
198                    self.index_row
199                ],
200                **kwargs
201            )
202
203            return self
204
205        if self.level is not None:
206            self.obj = PredictionInterval(
207                obj=self.obj, method=self.pi_method, level=self.level
208            )
209
210        self.obj.fit(scaled_Z, centered_y, **kwargs)
211
212        self.X_ = X
213
214        self.y_ = y
215
216        return self
217
218    def partial_fit(self, X, y, sample_weight=None, **kwargs):
219        """Partial fit custom model to training data (X, y).
220
221        Parameters:
222
223            X: {array-like}, shape = [n_samples, n_features]
224                Subset of training vectors, where n_samples is the number
225                of samples and n_features is the number of features.
226
227            y: array-like, shape = [n_samples]
228                Subset of target values.
229
230            **kwargs: additional parameters to be passed to
231                self.cook_training_set or self.obj.fit
232
233        Returns:
234
235            self: object
236
237        """
238
239        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
240
241        # if sample_weights, else: (must use self.row_index)
242        if sample_weight is not None:
243            try:
244                self.obj.partial_fit(
245                    scaled_Z,
246                    centered_y,
247                    sample_weight=np.ravel(sample_weight, order="C")[
248                        self.index_row
249                    ],
250                    **kwargs
251                )
252            except:
253                raise NotImplementedError
254
255            return self
256
257        try:
258            self.obj.partial_fit(scaled_Z, centered_y, **kwargs)
259        except:
260            raise NotImplementedError
261
262        self.X_ = X
263
264        self.y_ = y
265
266        return self
267
268    def predict(self, X, level=95, method=None, **kwargs):
269        """Predict test data X.
270
271        Parameters:
272
273            X: {array-like}, shape = [n_samples, n_features]
274                Training vectors, where n_samples is the number
275                of samples and n_features is the number of features.
276
277            level: int
278                Level of confidence (default = 95)
279
280            method: str
281                `None`, or 'splitconformal', 'localconformal'
282                prediction (if you specify `return_pi = True`)
283
284            **kwargs: additional parameters
285                    `return_pi = True` for conformal prediction,
286                    with `method` in ('splitconformal', 'localconformal')
287                    or `return_std = True` for `self.obj` in
288                    (`sklearn.linear_model.BayesianRidge`,
289                    `sklearn.linear_model.ARDRegressor`,
290                    `sklearn.gaussian_process.GaussianProcessRegressor`)`
291
292        Returns:
293
294            model predictions:
295                an array if uncertainty quantification is not requested,
296                  or a tuple if with prediction intervals and simulations
297                  if `return_std = True` (mean, standard deviation,
298                  lower and upper prediction interval) or `return_pi = True`
299                  ()
300
301        """
302
303        if "return_std" in kwargs:
304
305            alpha = 100 - level
306            pi_multiplier = norm.ppf(1 - alpha / 200)
307
308            if len(X.shape) == 1:
309
310                n_features = X.shape[0]
311                new_X = mo.rbind(
312                    X.reshape(1, n_features),
313                    np.ones(n_features).reshape(1, n_features),
314                )
315
316                mean_, std_ = self.obj.predict(
317                    self.cook_test_set(new_X, **kwargs), return_std=True
318                )[0]
319
320                preds = self.y_mean_ + mean_
321                lower = self.y_mean_ + (mean_ - pi_multiplier * std_)
322                upper = self.y_mean_ + (mean_ + pi_multiplier * std_)
323
324                return preds, std_, lower, upper
325
326            # len(X.shape) > 1
327            mean_, std_ = self.obj.predict(
328                self.cook_test_set(X, **kwargs), return_std=True
329            )
330
331            preds = self.y_mean_ + mean_
332            lower = self.y_mean_ + (mean_ - pi_multiplier * std_)
333            upper = self.y_mean_ + (mean_ + pi_multiplier * std_)
334
335            return preds, std_, lower, upper
336
337        if "return_pi" in kwargs:
338            assert method in (
339                "splitconformal",
340                "localconformal",
341            ), "method must be in ('splitconformal', 'localconformal')"
342            self.pi = PredictionInterval(
343                obj=self,
344                method=method,
345                level=level,
346                type_pi=self.type_pi,
347                replications=self.replications,
348                kernel=self.kernel,
349            )
350            self.pi.fit(self.X_, self.y_)
351            self.X_ = None
352            self.y_ = None
353            preds = self.pi.predict(X, return_pi=True)
354            return preds
355
356        # "return_std" not in kwargs
357        if len(X.shape) == 1:
358
359            n_features = X.shape[0]
360            new_X = mo.rbind(
361                X.reshape(1, n_features),
362                np.ones(n_features).reshape(1, n_features),
363            )
364
365            return (
366                self.y_mean_
367                + self.obj.predict(
368                    self.cook_test_set(new_X, **kwargs), **kwargs
369                )
370            )[0]
371
372        # len(X.shape) > 1
373        return self.y_mean_ + self.obj.predict(
374            self.cook_test_set(X, **kwargs), **kwargs
375        )

Custom Regression model

This class is used to 'augment' any regression model with transformed features.

Parameters:

obj: object
    any object containing a method fit (obj.fit()) and a method predict
    (obj.predict())

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original predictors are included (True) in model's
    fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

type_pi: str.
    type of prediction interval; currently "kde" (default).
    Used only in `self.predict`, for `self.replications` > 0 and `self.kernel`
    in ('gaussian', 'tophat'). Default is `None`.

replications: int.
    number of replications (if needed) for predictive simulation.
    Used only in `self.predict`, for `self.kernel` in ('gaussian',
    'tophat') and `self.type_pi = 'kde'`. Default is `None`.

kernel: str.
    the kernel to use for kernel density estimation (used for predictive
    simulation in `self.predict`, with `method='splitconformal'` and
    `type_pi = 'kde'`). Currently, either 'gaussian' or 'tophat'.

type_split: str.
    Type of splitting for conformal prediction. None (default), or
    "random" (random split of data) or "sequential" (sequential split of data)

col_sample: float
    percentage of covariates randomly chosen for training

row_sample: float
    percentage of rows chosen for training, by stratified bootstrapping

level: float
    confidence level for prediction intervals

pi_method: str
    method for prediction intervals: 'splitconformal' or 'localconformal'

seed: int
    reproducibility seed for nodes_sim=='uniform'

type_fit: str
    'regression'

backend: str
    "cpu" or "gpu" or "tpu"

Examples:

See https://thierrymoudiki.github.io/blog/2024/03/18/python/conformal-and-bayesian-regression

def fit(self, X, y, sample_weight=None, **kwargs):
169    def fit(self, X, y, sample_weight=None, **kwargs):
170        """Fit custom model to training data (X, y).
171
172        Parameters:
173
174            X: {array-like}, shape = [n_samples, n_features]
175                Training vectors, where n_samples is the number
176                of samples and n_features is the number of features.
177
178            y: array-like, shape = [n_samples]
179                Target values.
180
181            **kwargs: additional parameters to be passed to
182                self.cook_training_set or self.obj.fit
183
184        Returns:
185
186            self: object
187
188        """
189
190        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
191
192        # if sample_weights, else: (must use self.row_index)
193        if sample_weight is not None:
194            self.obj.fit(
195                scaled_Z,
196                centered_y,
197                sample_weight=np.ravel(sample_weight, order="C")[
198                    self.index_row
199                ],
200                **kwargs
201            )
202
203            return self
204
205        if self.level is not None:
206            self.obj = PredictionInterval(
207                obj=self.obj, method=self.pi_method, level=self.level
208            )
209
210        self.obj.fit(scaled_Z, centered_y, **kwargs)
211
212        self.X_ = X
213
214        self.y_ = y
215
216        return self

Fit custom model to training data (X, y).

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
    self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, level=95, method=None, **kwargs):
268    def predict(self, X, level=95, method=None, **kwargs):
269        """Predict test data X.
270
271        Parameters:
272
273            X: {array-like}, shape = [n_samples, n_features]
274                Training vectors, where n_samples is the number
275                of samples and n_features is the number of features.
276
277            level: int
278                Level of confidence (default = 95)
279
280            method: str
281                `None`, or 'splitconformal', 'localconformal'
282                prediction (if you specify `return_pi = True`)
283
284            **kwargs: additional parameters
285                    `return_pi = True` for conformal prediction,
286                    with `method` in ('splitconformal', 'localconformal')
287                    or `return_std = True` for `self.obj` in
288                    (`sklearn.linear_model.BayesianRidge`,
289                    `sklearn.linear_model.ARDRegressor`,
290                    `sklearn.gaussian_process.GaussianProcessRegressor`)`
291
292        Returns:
293
294            model predictions:
295                an array if uncertainty quantification is not requested,
296                  or a tuple if with prediction intervals and simulations
297                  if `return_std = True` (mean, standard deviation,
298                  lower and upper prediction interval) or `return_pi = True`
299                  ()
300
301        """
302
303        if "return_std" in kwargs:
304
305            alpha = 100 - level
306            pi_multiplier = norm.ppf(1 - alpha / 200)
307
308            if len(X.shape) == 1:
309
310                n_features = X.shape[0]
311                new_X = mo.rbind(
312                    X.reshape(1, n_features),
313                    np.ones(n_features).reshape(1, n_features),
314                )
315
316                mean_, std_ = self.obj.predict(
317                    self.cook_test_set(new_X, **kwargs), return_std=True
318                )[0]
319
320                preds = self.y_mean_ + mean_
321                lower = self.y_mean_ + (mean_ - pi_multiplier * std_)
322                upper = self.y_mean_ + (mean_ + pi_multiplier * std_)
323
324                return preds, std_, lower, upper
325
326            # len(X.shape) > 1
327            mean_, std_ = self.obj.predict(
328                self.cook_test_set(X, **kwargs), return_std=True
329            )
330
331            preds = self.y_mean_ + mean_
332            lower = self.y_mean_ + (mean_ - pi_multiplier * std_)
333            upper = self.y_mean_ + (mean_ + pi_multiplier * std_)
334
335            return preds, std_, lower, upper
336
337        if "return_pi" in kwargs:
338            assert method in (
339                "splitconformal",
340                "localconformal",
341            ), "method must be in ('splitconformal', 'localconformal')"
342            self.pi = PredictionInterval(
343                obj=self,
344                method=method,
345                level=level,
346                type_pi=self.type_pi,
347                replications=self.replications,
348                kernel=self.kernel,
349            )
350            self.pi.fit(self.X_, self.y_)
351            self.X_ = None
352            self.y_ = None
353            preds = self.pi.predict(X, return_pi=True)
354            return preds
355
356        # "return_std" not in kwargs
357        if len(X.shape) == 1:
358
359            n_features = X.shape[0]
360            new_X = mo.rbind(
361                X.reshape(1, n_features),
362                np.ones(n_features).reshape(1, n_features),
363            )
364
365            return (
366                self.y_mean_
367                + self.obj.predict(
368                    self.cook_test_set(new_X, **kwargs), **kwargs
369                )
370            )[0]
371
372        # len(X.shape) > 1
373        return self.y_mean_ + self.obj.predict(
374            self.cook_test_set(X, **kwargs), **kwargs
375        )

Predict test data X.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

level: int
    Level of confidence (default = 95)

method: str
    `None`, or 'splitconformal', 'localconformal'
    prediction (if you specify `return_pi = True`)

**kwargs: additional parameters
        `return_pi = True` for conformal prediction,
        with `method` in ('splitconformal', 'localconformal')
        or `return_std = True` for `self.obj` in
        (`sklearn.linear_model.BayesianRidge`,
        `sklearn.linear_model.ARDRegressor`,
        `sklearn.gaussian_process.GaussianProcessRegressor`)`

Returns:

model predictions:
    an array if uncertainty quantification is not requested,
      or a tuple if with prediction intervals and simulations
      if `return_std = True` (mean, standard deviation,
      lower and upper prediction interval) or `return_pi = True`
      ()
class DeepClassifier(nnetsauce.CustomClassifier, sklearn.base.ClassifierMixin):
 26class DeepClassifier(CustomClassifier, ClassifierMixin):
 27    """
 28    Deep Classifier
 29
 30    Parameters:
 31
 32        obj: an object
 33            A base learner, see also https://www.researchgate.net/publication/380701207_Deep_Quasi-Randomized_neural_Networks_for_classification
 34
 35        n_layers: int (default=3)
 36            Number of layers. `n_layers = 1` is a simple `CustomClassifier`
 37
 38        verbose : int, optional (default=0)
 39            Monitor progress when fitting.
 40
 41        All the other parameters are nnetsauce `CustomClassifier`'s
 42
 43    Examples:
 44
 45        ```python
 46        import nnetsauce as ns
 47        from sklearn.datasets import load_breast_cancer
 48        from sklearn.model_selection import train_test_split
 49        from sklearn.linear_model import LogisticRegressionCV
 50        data = load_breast_cancer()
 51        X = data.data
 52        y= data.target
 53        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123)
 54        obj = LogisticRegressionCV()
 55        clf = ns.DeepClassifier(obj)
 56        clf.fit(X_train, y_train)
 57        print(clf.score(clf.predict(X_test), y_test))
 58        ```
 59    """
 60
 61    def __init__(
 62        self,
 63        obj,
 64        # Defining depth
 65        n_layers=3,
 66        verbose=0,
 67        # CustomClassifier attributes
 68        n_hidden_features=5,
 69        activation_name="relu",
 70        a=0.01,
 71        nodes_sim="sobol",
 72        bias=True,
 73        dropout=0,
 74        direct_link=True,
 75        n_clusters=2,
 76        cluster_encode=True,
 77        type_clust="kmeans",
 78        type_scaling=("std", "std", "std"),
 79        col_sample=1,
 80        row_sample=1,
 81        level=None,
 82        pi_method="icp",
 83        seed=123,
 84        backend="cpu",
 85    ):
 86        super().__init__(
 87            obj=obj,
 88            n_hidden_features=n_hidden_features,
 89            activation_name=activation_name,
 90            a=a,
 91            nodes_sim=nodes_sim,
 92            bias=bias,
 93            dropout=dropout,
 94            direct_link=direct_link,
 95            n_clusters=n_clusters,
 96            cluster_encode=cluster_encode,
 97            type_clust=type_clust,
 98            type_scaling=type_scaling,
 99            col_sample=col_sample,
100            row_sample=row_sample,
101            level=level,
102            pi_method=pi_method,
103            seed=seed,
104            backend=backend,
105        )
106
107        assert n_layers >= 1, "must have n_layers >= 1"
108
109        self.stacked_obj = obj
110        self.verbose = verbose
111        self.n_layers = n_layers
112
113    def fit(self, X, y):
114        """Fit Classification algorithms to X and y.
115        Parameters
116        ----------
117        X : array-like,
118            Training vectors, where rows is the number of samples
119            and columns is the number of features.
120        y : array-like,
121            Training vectors, where rows is the number of samples
122            and columns is the number of features.
123        Returns
124        -------
125        A fitted object
126        """
127
128        if isinstance(X, np.ndarray):
129            X = pd.DataFrame(X)
130
131        self.classes_ = np.unique(y)  # for compatibility with sklearn
132        self.n_classes_ = len(self.classes_)  # for compatibility with sklearn
133
134        # init layer
135        self.stacked_obj = CustomClassifier(
136            obj=self.stacked_obj,
137            n_hidden_features=self.n_hidden_features,
138            activation_name=self.activation_name,
139            a=self.a,
140            nodes_sim=self.nodes_sim,
141            bias=self.bias,
142            dropout=self.dropout,
143            direct_link=self.direct_link,
144            n_clusters=self.n_clusters,
145            cluster_encode=self.cluster_encode,
146            type_clust=self.type_clust,
147            type_scaling=self.type_scaling,
148            col_sample=self.col_sample,
149            row_sample=self.row_sample,
150            seed=self.seed,
151            backend=self.backend,
152        )
153
154        self.stacked_obj.fit(X, y)
155
156        if self.verbose > 0:
157            iterator = tqdm(range(self.n_layers - 1))
158        else:
159            iterator = range(self.n_layers - 1)
160
161        for _ in iterator:
162            self.stacked_obj = deepcopy(
163                CustomClassifier(
164                    obj=self.stacked_obj,
165                    n_hidden_features=self.n_hidden_features,
166                    activation_name=self.activation_name,
167                    a=self.a,
168                    nodes_sim=self.nodes_sim,
169                    bias=self.bias,
170                    dropout=self.dropout,
171                    direct_link=self.direct_link,
172                    n_clusters=self.n_clusters,
173                    cluster_encode=self.cluster_encode,
174                    type_clust=self.type_clust,
175                    type_scaling=self.type_scaling,
176                    col_sample=self.col_sample,
177                    row_sample=self.row_sample,
178                    seed=self.seed,
179                    backend=self.backend,
180                )
181            )
182
183        if self.level is not None:
184            self.stacked_obj = PredictionSet(
185                obj=self.stacked_obj, method=self.pi_method, level=self.level
186            )
187
188        self.stacked_obj.fit(X, y)
189
190        self.obj = deepcopy(self.stacked_obj)
191
192        return self.obj
193
194    def predict(self, X):
195        return self.obj.predict(X)
196
197    def predict_proba(self, X):
198        return self.obj.predict_proba(X)
199
200    def score(self, X, y, scoring=None):
201        return self.obj.score(X, y, scoring)
202
203    def cross_val_optim(
204        self,
205        X_train,
206        y_train,
207        X_test=None,
208        y_test=None,
209        scoring="accuracy",
210        surrogate_obj=None,
211        cv=5,
212        n_jobs=None,
213        n_init=10,
214        n_iter=190,
215        abs_tol=1e-3,
216        verbose=2,
217        seed=123,
218        **kwargs,
219    ):
220        """Cross-validation function and hyperparameters' search
221
222        Parameters:
223
224            X_train: array-like,
225                Training vectors, where rows is the number of samples
226                and columns is the number of features.
227
228            y_train: array-like,
229                Training vectors, where rows is the number of samples
230                and columns is the number of features.
231
232            X_test: array-like,
233                Testing vectors, where rows is the number of samples
234                and columns is the number of features.
235
236            y_test: array-like,
237                Testing vectors, where rows is the number of samples
238                and columns is the number of features.
239
240            scoring: str
241                scoring metric; see https://scikit-learn.org/stable/modules/model_evaluation.html#the-scoring-parameter-defining-model-evaluation-rules
242
243            surrogate_obj: an object;
244                An ML model for estimating the uncertainty around the objective function
245
246            cv: int;
247                number of cross-validation folds
248
249            n_jobs: int;
250                number of jobs for parallel execution
251
252            n_init: an integer;
253                number of points in the initial setting, when `x_init` and `y_init` are not provided
254
255            n_iter: an integer;
256                number of iterations of the minimization algorithm
257
258            abs_tol: a float;
259                tolerance for convergence of the optimizer (early stopping based on acquisition function)
260
261            verbose: int
262                controls verbosity
263
264            seed: int
265                reproducibility seed
266
267            **kwargs: dict
268                additional parameters to be passed to the estimator
269
270        Examples:
271
272            ```python
273            ```
274        """
275
276        num_to_activation_name = {1: "relu", 2: "sigmoid", 3: "tanh"}
277        num_to_nodes_sim = {1: "sobol", 2: "uniform", 3: "hammersley"}
278        num_to_type_clust = {1: "kmeans", 2: "gmm"}
279
280        def deepclassifier_cv(
281            X_train,
282            y_train,
283            # Defining depth
284            n_layers=3,
285            # CustomClassifier attributes
286            n_hidden_features=5,
287            activation_name="relu",
288            nodes_sim="sobol",
289            dropout=0,
290            n_clusters=2,
291            type_clust="kmeans",
292            cv=5,
293            n_jobs=None,
294            scoring="accuracy",
295            seed=123,
296        ):
297            self.set_params(
298                **{
299                    "n_layers": n_layers,
300                    # CustomClassifier attributes
301                    "n_hidden_features": n_hidden_features,
302                    "activation_name": activation_name,
303                    "nodes_sim": nodes_sim,
304                    "dropout": dropout,
305                    "n_clusters": n_clusters,
306                    "type_clust": type_clust,
307                    **kwargs,
308                }
309            )
310            return -cross_val_score(
311                estimator=self,
312                X=X_train,
313                y=y_train,
314                scoring=scoring,
315                cv=cv,
316                n_jobs=n_jobs,
317                verbose=0,
318            ).mean()
319
320        # objective function for hyperparams tuning
321        def crossval_objective(xx):
322            return deepclassifier_cv(
323                X_train=X_train,
324                y_train=y_train,
325                # Defining depth
326                n_layers=int(np.ceil(xx[0])),
327                # CustomClassifier attributes
328                n_hidden_features=int(np.ceil(xx[1])),
329                activation_name=num_to_activation_name[np.ceil(xx[2])],
330                nodes_sim=num_to_nodes_sim[int(np.ceil(xx[3]))],
331                dropout=xx[4],
332                n_clusters=int(np.ceil(xx[5])),
333                type_clust=num_to_type_clust[int(np.ceil(xx[6]))],
334                cv=cv,
335                n_jobs=n_jobs,
336                scoring=scoring,
337                seed=seed,
338            )
339
340        if surrogate_obj is None:
341            gp_opt = gp.GPOpt(
342                objective_func=crossval_objective,
343                lower_bound=np.array([0, 3, 0, 0, 0.0, 0, 0]),
344                upper_bound=np.array([5, 100, 3, 3, 0.4, 5, 2]),
345                params_names=[
346                    "n_layers",
347                    # CustomClassifier attributes
348                    "n_hidden_features",
349                    "activation_name",
350                    "nodes_sim",
351                    "dropout",
352                    "n_clusters",
353                    "type_clust",
354                ],
355                method="bayesian",
356                n_init=n_init,
357                n_iter=n_iter,
358                seed=seed,
359            )
360        else:
361            gp_opt = gp.GPOpt(
362                objective_func=crossval_objective,
363                lower_bound=np.array([0, 3, 0, 0, 0.0, 0, 0]),
364                upper_bound=np.array([5, 100, 3, 3, 0.4, 5, 2]),
365                params_names=[
366                    "n_layers",
367                    # CustomClassifier attributes
368                    "n_hidden_features",
369                    "activation_name",
370                    "nodes_sim",
371                    "dropout",
372                    "n_clusters",
373                    "type_clust",
374                ],
375                acquisition="ucb",
376                method="splitconformal",
377                surrogate_obj=ns.PredictionInterval(
378                    obj=surrogate_obj, method="splitconformal"
379                ),
380                n_init=n_init,
381                n_iter=n_iter,
382                seed=seed,
383            )
384
385        res = gp_opt.optimize(verbose=verbose, abs_tol=abs_tol)
386        res.best_params["n_layers"] = int(np.ceil(res.best_params["n_layers"]))
387        res.best_params["n_hidden_features"] = int(
388            np.ceil(res.best_params["n_hidden_features"])
389        )
390        res.best_params["activation_name"] = num_to_activation_name[
391            np.ceil(res.best_params["activation_name"])
392        ]
393        res.best_params["nodes_sim"] = num_to_nodes_sim[
394            int(np.ceil(res.best_params["nodes_sim"]))
395        ]
396        res.best_params["dropout"] = res.best_params["dropout"]
397        res.best_params["n_clusters"] = int(
398            np.ceil(res.best_params["n_clusters"])
399        )
400        res.best_params["type_clust"] = num_to_type_clust[
401            int(np.ceil(res.best_params["type_clust"]))
402        ]
403
404        # out-of-sample error
405        if X_test is not None and y_test is not None:
406            self.set_params(**res.best_params, verbose=0, seed=seed)
407            preds = self.fit(X_train, y_train).predict(X_test)
408            # check error on y_test
409            oos_err = getattr(metrics, scoring + "_score")(
410                y_true=y_test, y_pred=preds
411            )
412            result = namedtuple("result", res._fields + ("test_" + scoring,))
413            return result(*res, oos_err)
414        else:
415            return res
416
417    def lazy_cross_val_optim(
418        self,
419        X_train,
420        y_train,
421        X_test=None,
422        y_test=None,
423        scoring="accuracy",
424        surrogate_objs=None,
425        customize=False,
426        cv=5,
427        n_jobs=None,
428        n_init=10,
429        n_iter=190,
430        abs_tol=1e-3,
431        verbose=1,
432        seed=123,
433    ):
434        """Automated Cross-validation function and hyperparameters' search using multiple surrogates
435
436        Parameters:
437
438            X_train: array-like,
439                Training vectors, where rows is the number of samples
440                and columns is the number of features.
441
442            y_train: array-like,
443                Training vectors, where rows is the number of samples
444                and columns is the number of features.
445
446            X_test: array-like,
447                Testing vectors, where rows is the number of samples
448                and columns is the number of features.
449
450            y_test: array-like,
451                Testing vectors, where rows is the number of samples
452                and columns is the number of features.
453
454            scoring: str
455                scoring metric; see https://scikit-learn.org/stable/modules/model_evaluation.html#the-scoring-parameter-defining-model-evaluation-rules
456
457            surrogate_objs: object names as a list of strings;
458                ML models for estimating the uncertainty around the objective function
459
460            customize: boolean
461                if True, the surrogate is transformed into a quasi-randomized network (default is False)
462
463            cv: int;
464                number of cross-validation folds
465
466            n_jobs: int;
467                number of jobs for parallel execution
468
469            n_init: an integer;
470                number of points in the initial setting, when `x_init` and `y_init` are not provided
471
472            n_iter: an integer;
473                number of iterations of the minimization algorithm
474
475            abs_tol: a float;
476                tolerance for convergence of the optimizer (early stopping based on acquisition function)
477
478            verbose: int
479                controls verbosity
480
481            seed: int
482                reproducibility seed
483
484        Examples:
485
486            ```python
487            ```
488        """
489
490        removed_regressors = [
491            "TheilSenRegressor",
492            "ARDRegression",
493            "CCA",
494            "GaussianProcessRegressor",
495            "GradientBoostingRegressor",
496            "HistGradientBoostingRegressor",
497            "IsotonicRegression",
498            "MultiOutputRegressor",
499            "MultiTaskElasticNet",
500            "MultiTaskElasticNetCV",
501            "MultiTaskLasso",
502            "MultiTaskLassoCV",
503            "OrthogonalMatchingPursuit",
504            "OrthogonalMatchingPursuitCV",
505            "PLSCanonical",
506            "PLSRegression",
507            "RadiusNeighborsRegressor",
508            "RegressorChain",
509            "StackingRegressor",
510            "VotingRegressor",
511        ]
512
513        results = []
514
515        for est in all_estimators():
516
517            if surrogate_objs is None:
518
519                if issubclass(est[1], RegressorMixin) and (
520                    est[0] not in removed_regressors
521                ):
522                    try:
523                        if customize == True:
524                            print(f"\n surrogate: CustomRegressor({est[0]})")
525                            surr_obj = ns.CustomRegressor(obj=est[1]())
526                        else:
527                            print(f"\n surrogate: {est[0]}")
528                            surr_obj = est[1]()
529                        res = self.cross_val_optim(
530                            X_train=X_train,
531                            y_train=y_train,
532                            X_test=X_test,
533                            y_test=y_test,
534                            surrogate_obj=surr_obj,
535                            cv=cv,
536                            n_jobs=n_jobs,
537                            scoring=scoring,
538                            n_init=n_init,
539                            n_iter=n_iter,
540                            abs_tol=abs_tol,
541                            verbose=verbose,
542                            seed=seed,
543                        )
544                        print(f"\n result: {res}")
545                        if customize == True:
546                            results.append((f"CustomRegressor({est[0]})", res))
547                        else:
548                            results.append((est[0], res))
549                    except:
550                        pass
551
552            else:
553
554                if (
555                    issubclass(est[1], RegressorMixin)
556                    and (est[0] not in removed_regressors)
557                    and est[0] in surrogate_objs
558                ):
559                    try:
560                        if customize == True:
561                            print(f"\n surrogate: CustomRegressor({est[0]})")
562                            surr_obj = ns.CustomRegressor(obj=est[1]())
563                        else:
564                            print(f"\n surrogate: {est[0]}")
565                            surr_obj = est[1]()
566                        res = self.cross_val_optim(
567                            X_train=X_train,
568                            y_train=y_train,
569                            X_test=X_test,
570                            y_test=y_test,
571                            surrogate_obj=surr_obj,
572                            cv=cv,
573                            n_jobs=n_jobs,
574                            scoring=scoring,
575                            n_init=n_init,
576                            n_iter=n_iter,
577                            abs_tol=abs_tol,
578                            verbose=verbose,
579                            seed=seed,
580                        )
581                        print(f"\n result: {res}")
582                        if customize == True:
583                            results.append((f"CustomRegressor({est[0]})", res))
584                        else:
585                            results.append((est[0], res))
586                    except:
587                        pass
588
589        return results

Deep Classifier

Parameters:

obj: an object
    A base learner, see also https://www.researchgate.net/publication/380701207_Deep_Quasi-Randomized_neural_Networks_for_classification

n_layers: int (default=3)
    Number of layers. `n_layers = 1` is a simple `CustomClassifier`

verbose : int, optional (default=0)
    Monitor progress when fitting.

All the other parameters are nnetsauce `CustomClassifier`'s

Examples:

import nnetsauce as ns
    from sklearn.datasets import load_breast_cancer
    from sklearn.model_selection import train_test_split
    from sklearn.linear_model import LogisticRegressionCV
    data = load_breast_cancer()
    X = data.data
    y= data.target
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123)
    obj = LogisticRegressionCV()
    clf = ns.DeepClassifier(obj)
    clf.fit(X_train, y_train)
    print(clf.score(clf.predict(X_test), y_test))
    

def fit(self, X, y):
113    def fit(self, X, y):
114        """Fit Classification algorithms to X and y.
115        Parameters
116        ----------
117        X : array-like,
118            Training vectors, where rows is the number of samples
119            and columns is the number of features.
120        y : array-like,
121            Training vectors, where rows is the number of samples
122            and columns is the number of features.
123        Returns
124        -------
125        A fitted object
126        """
127
128        if isinstance(X, np.ndarray):
129            X = pd.DataFrame(X)
130
131        self.classes_ = np.unique(y)  # for compatibility with sklearn
132        self.n_classes_ = len(self.classes_)  # for compatibility with sklearn
133
134        # init layer
135        self.stacked_obj = CustomClassifier(
136            obj=self.stacked_obj,
137            n_hidden_features=self.n_hidden_features,
138            activation_name=self.activation_name,
139            a=self.a,
140            nodes_sim=self.nodes_sim,
141            bias=self.bias,
142            dropout=self.dropout,
143            direct_link=self.direct_link,
144            n_clusters=self.n_clusters,
145            cluster_encode=self.cluster_encode,
146            type_clust=self.type_clust,
147            type_scaling=self.type_scaling,
148            col_sample=self.col_sample,
149            row_sample=self.row_sample,
150            seed=self.seed,
151            backend=self.backend,
152        )
153
154        self.stacked_obj.fit(X, y)
155
156        if self.verbose > 0:
157            iterator = tqdm(range(self.n_layers - 1))
158        else:
159            iterator = range(self.n_layers - 1)
160
161        for _ in iterator:
162            self.stacked_obj = deepcopy(
163                CustomClassifier(
164                    obj=self.stacked_obj,
165                    n_hidden_features=self.n_hidden_features,
166                    activation_name=self.activation_name,
167                    a=self.a,
168                    nodes_sim=self.nodes_sim,
169                    bias=self.bias,
170                    dropout=self.dropout,
171                    direct_link=self.direct_link,
172                    n_clusters=self.n_clusters,
173                    cluster_encode=self.cluster_encode,
174                    type_clust=self.type_clust,
175                    type_scaling=self.type_scaling,
176                    col_sample=self.col_sample,
177                    row_sample=self.row_sample,
178                    seed=self.seed,
179                    backend=self.backend,
180                )
181            )
182
183        if self.level is not None:
184            self.stacked_obj = PredictionSet(
185                obj=self.stacked_obj, method=self.pi_method, level=self.level
186            )
187
188        self.stacked_obj.fit(X, y)
189
190        self.obj = deepcopy(self.stacked_obj)
191
192        return self.obj

Fit Classification algorithms to X and y.

Parameters

X : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y : array-like, Training vectors, where rows is the number of samples and columns is the number of features.

Returns

A fitted object

def predict(self, X):
194    def predict(self, X):
195        return self.obj.predict(X)

Predict test data X.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
def predict_proba(self, X):
197    def predict_proba(self, X):
198        return self.obj.predict_proba(X)

Predict probabilities for test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

probability estimates for test data: {array-like}
def score(self, X, y, scoring=None):
200    def score(self, X, y, scoring=None):
201        return self.obj.score(X, y, scoring)

Score the model on test set features X and response y.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

scoring: str
    must be in ('explained_variance', 'neg_mean_absolute_error',
                'neg_mean_squared_error', 'neg_mean_squared_log_error',
                'neg_median_absolute_error', 'r2')

**kwargs: additional parameters to be passed to scoring functions

Returns:

model scores: {array-like}

class DeepRegressor(nnetsauce.CustomRegressor, sklearn.base.RegressorMixin):
 13class DeepRegressor(CustomRegressor, RegressorMixin):
 14    """
 15    Deep Regressor
 16
 17    Parameters:
 18
 19        obj: an object
 20            A base learner, see also https://www.researchgate.net/publication/380701207_Deep_Quasi-Randomized_neural_Networks_for_classification
 21
 22        verbose : int, optional (default=0)
 23            Monitor progress when fitting.
 24
 25        n_layers: int (default=3)
 26            Number of layers. `n_layers = 1` is a simple `CustomRegressor`
 27
 28        All the other parameters are nnetsauce `CustomRegressor`'s
 29
 30    Examples:
 31
 32        ```python
 33        import nnetsauce as ns
 34        from sklearn.datasets import load_diabetes
 35        from sklearn.model_selection import train_test_split
 36        from sklearn.linear_model import RidgeCV
 37        data = load_diabetes()
 38        X = data.data
 39        y= data.target
 40        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123)
 41        obj = RidgeCV()
 42        clf = ns.DeepRegressor(obj)
 43        clf.fit(X_train, y_train)
 44        print(clf.score(clf.predict(X_test), y_test))
 45        ```
 46
 47    """
 48
 49    def __init__(
 50        self,
 51        obj,
 52        # Defining depth
 53        n_layers=3,
 54        verbose=0,
 55        # CustomRegressor attributes
 56        n_hidden_features=5,
 57        activation_name="relu",
 58        a=0.01,
 59        nodes_sim="sobol",
 60        bias=True,
 61        dropout=0,
 62        direct_link=True,
 63        n_clusters=2,
 64        cluster_encode=True,
 65        type_clust="kmeans",
 66        type_scaling=("std", "std", "std"),
 67        col_sample=1,
 68        row_sample=1,
 69        level=None,
 70        pi_method="splitconformal",
 71        seed=123,
 72        backend="cpu",
 73    ):
 74        super().__init__(
 75            obj=obj,
 76            n_hidden_features=n_hidden_features,
 77            activation_name=activation_name,
 78            a=a,
 79            nodes_sim=nodes_sim,
 80            bias=bias,
 81            dropout=dropout,
 82            direct_link=direct_link,
 83            n_clusters=n_clusters,
 84            cluster_encode=cluster_encode,
 85            type_clust=type_clust,
 86            type_scaling=type_scaling,
 87            col_sample=col_sample,
 88            row_sample=row_sample,
 89            level=level,
 90            pi_method=pi_method,
 91            seed=seed,
 92            backend=backend,
 93        )
 94
 95        assert n_layers >= 1, "must have n_layers >= 1"
 96
 97        self.stacked_obj = obj
 98        self.verbose = verbose
 99        self.n_layers = n_layers
100        self.level = level
101        self.pi_method = pi_method
102
103    def fit(self, X, y):
104        """Fit Regression algorithms to X and y.
105        Parameters
106        ----------
107        X : array-like,
108            Training vectors, where rows is the number of samples
109            and columns is the number of features.
110        y : array-like,
111            Training vectors, where rows is the number of samples
112            and columns is the number of features.
113        Returns
114        -------
115        A fitted object
116        """
117
118        if isinstance(X, np.ndarray):
119            X = pd.DataFrame(X)
120
121        # init layer
122        self.stacked_obj = CustomRegressor(
123            obj=self.stacked_obj,
124            n_hidden_features=self.n_hidden_features,
125            activation_name=self.activation_name,
126            a=self.a,
127            nodes_sim=self.nodes_sim,
128            bias=self.bias,
129            dropout=self.dropout,
130            direct_link=self.direct_link,
131            n_clusters=self.n_clusters,
132            cluster_encode=self.cluster_encode,
133            type_clust=self.type_clust,
134            type_scaling=self.type_scaling,
135            col_sample=self.col_sample,
136            row_sample=self.row_sample,
137            seed=self.seed,
138            backend=self.backend,
139        )
140
141        if self.verbose > 0:
142            iterator = tqdm(range(self.n_layers - 1))
143        else:
144            iterator = range(self.n_layers - 1)
145
146        for _ in iterator:
147            self.stacked_obj = deepcopy(
148                CustomRegressor(
149                    obj=self.stacked_obj,
150                    n_hidden_features=self.n_hidden_features,
151                    activation_name=self.activation_name,
152                    a=self.a,
153                    nodes_sim=self.nodes_sim,
154                    bias=self.bias,
155                    dropout=self.dropout,
156                    direct_link=self.direct_link,
157                    n_clusters=self.n_clusters,
158                    cluster_encode=self.cluster_encode,
159                    type_clust=self.type_clust,
160                    type_scaling=self.type_scaling,
161                    col_sample=self.col_sample,
162                    row_sample=self.row_sample,
163                    seed=self.seed,
164                    backend=self.backend,
165                )
166            )
167        if self.level is not None:
168            self.stacked_obj = PredictionInterval(
169                obj=self.stacked_obj, method=self.pi_method, level=self.level
170            )
171        self.stacked_obj.fit(X, y)
172
173        self.obj = deepcopy(self.stacked_obj)
174
175        return self.obj
176
177    def predict(self, X, **kwargs):
178        if self.level is not None:
179            return self.obj.predict(X, return_pi=True)
180        return self.obj.predict(X, **kwargs)
181
182    def score(self, X, y, scoring=None):
183        return self.obj.score(X, y, scoring)

Deep Regressor

Parameters:

obj: an object
    A base learner, see also https://www.researchgate.net/publication/380701207_Deep_Quasi-Randomized_neural_Networks_for_classification

verbose : int, optional (default=0)
    Monitor progress when fitting.

n_layers: int (default=3)
    Number of layers. `n_layers = 1` is a simple `CustomRegressor`

All the other parameters are nnetsauce `CustomRegressor`'s

Examples:

import nnetsauce as ns
    from sklearn.datasets import load_diabetes
    from sklearn.model_selection import train_test_split
    from sklearn.linear_model import RidgeCV
    data = load_diabetes()
    X = data.data
    y= data.target
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123)
    obj = RidgeCV()
    clf = ns.DeepRegressor(obj)
    clf.fit(X_train, y_train)
    print(clf.score(clf.predict(X_test), y_test))
    

def fit(self, X, y):
103    def fit(self, X, y):
104        """Fit Regression algorithms to X and y.
105        Parameters
106        ----------
107        X : array-like,
108            Training vectors, where rows is the number of samples
109            and columns is the number of features.
110        y : array-like,
111            Training vectors, where rows is the number of samples
112            and columns is the number of features.
113        Returns
114        -------
115        A fitted object
116        """
117
118        if isinstance(X, np.ndarray):
119            X = pd.DataFrame(X)
120
121        # init layer
122        self.stacked_obj = CustomRegressor(
123            obj=self.stacked_obj,
124            n_hidden_features=self.n_hidden_features,
125            activation_name=self.activation_name,
126            a=self.a,
127            nodes_sim=self.nodes_sim,
128            bias=self.bias,
129            dropout=self.dropout,
130            direct_link=self.direct_link,
131            n_clusters=self.n_clusters,
132            cluster_encode=self.cluster_encode,
133            type_clust=self.type_clust,
134            type_scaling=self.type_scaling,
135            col_sample=self.col_sample,
136            row_sample=self.row_sample,
137            seed=self.seed,
138            backend=self.backend,
139        )
140
141        if self.verbose > 0:
142            iterator = tqdm(range(self.n_layers - 1))
143        else:
144            iterator = range(self.n_layers - 1)
145
146        for _ in iterator:
147            self.stacked_obj = deepcopy(
148                CustomRegressor(
149                    obj=self.stacked_obj,
150                    n_hidden_features=self.n_hidden_features,
151                    activation_name=self.activation_name,
152                    a=self.a,
153                    nodes_sim=self.nodes_sim,
154                    bias=self.bias,
155                    dropout=self.dropout,
156                    direct_link=self.direct_link,
157                    n_clusters=self.n_clusters,
158                    cluster_encode=self.cluster_encode,
159                    type_clust=self.type_clust,
160                    type_scaling=self.type_scaling,
161                    col_sample=self.col_sample,
162                    row_sample=self.row_sample,
163                    seed=self.seed,
164                    backend=self.backend,
165                )
166            )
167        if self.level is not None:
168            self.stacked_obj = PredictionInterval(
169                obj=self.stacked_obj, method=self.pi_method, level=self.level
170            )
171        self.stacked_obj.fit(X, y)
172
173        self.obj = deepcopy(self.stacked_obj)
174
175        return self.obj

Fit Regression algorithms to X and y.

Parameters

X : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y : array-like, Training vectors, where rows is the number of samples and columns is the number of features.

Returns

A fitted object

def predict(self, X, **kwargs):
177    def predict(self, X, **kwargs):
178        if self.level is not None:
179            return self.obj.predict(X, return_pi=True)
180        return self.obj.predict(X, **kwargs)

Predict test data X.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

level: int
    Level of confidence (default = 95)

method: str
    `None`, or 'splitconformal', 'localconformal'
    prediction (if you specify `return_pi = True`)

**kwargs: additional parameters
        `return_pi = True` for conformal prediction,
        with `method` in ('splitconformal', 'localconformal')
        or `return_std = True` for `self.obj` in
        (`sklearn.linear_model.BayesianRidge`,
        `sklearn.linear_model.ARDRegressor`,
        `sklearn.gaussian_process.GaussianProcessRegressor`)`

Returns:

model predictions:
    an array if uncertainty quantification is not requested,
      or a tuple if with prediction intervals and simulations
      if `return_std = True` (mean, standard deviation,
      lower and upper prediction interval) or `return_pi = True`
      ()
def score(self, X, y, scoring=None):
182    def score(self, X, y, scoring=None):
183        return self.obj.score(X, y, scoring)

Score the model on test set features X and response y.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

scoring: str
    must be in ('explained_variance', 'neg_mean_absolute_error',
                'neg_mean_squared_error', 'neg_mean_squared_log_error',
                'neg_median_absolute_error', 'r2')

**kwargs: additional parameters to be passed to scoring functions

Returns:

model scores: {array-like}

class DeepMTS(nnetsauce.MTS):
 10class DeepMTS(MTS):
 11    """Univariate and multivariate time series (DeepMTS) forecasting with Quasi-Randomized networks (Work in progress /!\)
 12
 13    Parameters:
 14
 15        obj: object.
 16            any object containing a method fit (obj.fit()) and a method predict
 17            (obj.predict()).
 18
 19        n_layers: int.
 20            number of layers in the neural network.
 21
 22        n_hidden_features: int.
 23            number of nodes in the hidden layer.
 24
 25        activation_name: str.
 26            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'.
 27
 28        a: float.
 29            hyperparameter for 'prelu' or 'elu' activation function.
 30
 31        nodes_sim: str.
 32            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 33            'uniform'.
 34
 35        bias: boolean.
 36            indicates if the hidden layer contains a bias term (True) or not
 37            (False).
 38
 39        dropout: float.
 40            regularization parameter; (random) percentage of nodes dropped out
 41            of the training.
 42
 43        direct_link: boolean.
 44            indicates if the original predictors are included (True) in model's fitting or not (False).
 45
 46        n_clusters: int.
 47            number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering).
 48
 49        cluster_encode: bool.
 50            defines how the variable containing clusters is treated (default is one-hot)
 51            if `False`, then labels are used, without one-hot encoding.
 52
 53        type_clust: str.
 54            type of clustering method: currently k-means ('kmeans') or Gaussian
 55            Mixture Model ('gmm').
 56
 57        type_scaling: a tuple of 3 strings.
 58            scaling methods for inputs, hidden layer, and clustering respectively
 59            (and when relevant).
 60            Currently available: standardization ('std') or MinMax scaling ('minmax').
 61
 62        lags: int.
 63            number of lags used for each time series.
 64
 65        type_pi: str.
 66            type of prediction interval; currently:
 67            - "gaussian": simple, fast, but: assumes stationarity of Gaussian in-sample residuals and independence in the multivariate case
 68            - "kde": based on Kernel Density Estimation of in-sample residuals
 69            - "bootstrap": based on independent bootstrap of in-sample residuals
 70            - "block-bootstrap": based on basic block bootstrap of in-sample residuals
 71            - "scp-kde": Sequential split conformal prediction with Kernel Density Estimation of calibrated residuals
 72            - "scp-bootstrap": Sequential split conformal prediction with independent bootstrap of calibrated residuals
 73            - "scp-block-bootstrap": Sequential split conformal prediction with basic block bootstrap of calibrated residuals
 74            - "scp2-kde": Sequential split conformal prediction with Kernel Density Estimation of standardized calibrated residuals
 75            - "scp2-bootstrap": Sequential split conformal prediction with independent bootstrap of standardized calibrated residuals
 76            - "scp2-block-bootstrap": Sequential split conformal prediction with basic block bootstrap of standardized calibrated residuals
 77
 78        block_size: int.
 79            size of block for 'type_pi' in ("block-bootstrap", "scp-block-bootstrap", "scp2-block-bootstrap").
 80            Default is round(3.15*(n_residuals^1/3))
 81
 82        replications: int.
 83            number of replications (if needed, for predictive simulation). Default is 'None'.
 84
 85        kernel: str.
 86            the kernel to use for residuals density estimation (used for predictive simulation). Currently, either 'gaussian' or 'tophat'.
 87
 88        agg: str.
 89            either "mean" or "median" for simulation of bootstrap aggregating
 90
 91        seed: int.
 92            reproducibility seed for nodes_sim=='uniform' or predictive simulation.
 93
 94        backend: str.
 95            "cpu" or "gpu" or "tpu".
 96
 97        verbose: int.
 98            0: not printing; 1: printing
 99
100        show_progress: bool.
101            True: progress bar when fitting each series; False: no progress bar when fitting each series
102
103    Attributes:
104
105        fit_objs_: dict
106            objects adjusted to each individual time series
107
108        y_: {array-like}
109            DeepMTS responses (most recent observations first)
110
111        X_: {array-like}
112            DeepMTS lags
113
114        xreg_: {array-like}
115            external regressors
116
117        y_means_: dict
118            a dictionary of each series mean values
119
120        preds_: {array-like}
121            successive model predictions
122
123        preds_std_: {array-like}
124            standard deviation around the predictions
125
126        return_std_: boolean
127            return uncertainty or not (set in predict)
128
129        df_: data frame
130            the input data frame, in case a data.frame is provided to `fit`
131
132    Examples:
133
134    Example 1:
135
136        ```python
137        import nnetsauce as ns
138        import numpy as np
139        from sklearn import linear_model
140        np.random.seed(123)
141
142        M = np.random.rand(10, 3)
143        M[:,0] = 10*M[:,0]
144        M[:,2] = 25*M[:,2]
145        print(M)
146
147        # Adjust Bayesian Ridge
148        regr4 = linear_model.BayesianRidge()
149        obj_DeepMTS = ns.DeepMTS(regr4, lags = 1, n_hidden_features=5)
150        obj_DeepMTS.fit(M)
151        print(obj_DeepMTS.predict())
152
153        # with credible intervals
154        print(obj_DeepMTS.predict(return_std=True, level=80))
155
156        print(obj_DeepMTS.predict(return_std=True, level=95))
157        ```
158
159    Example 2:
160
161        ```python
162        import nnetsauce as ns
163        import numpy as np
164        from sklearn import linear_model
165
166        dataset = {
167        'date' : ['2001-01-01', '2002-01-01', '2003-01-01', '2004-01-01', '2005-01-01'],
168        'series1' : [34, 30, 35.6, 33.3, 38.1],
169        'series2' : [4, 5.5, 5.6, 6.3, 5.1],
170        'series3' : [100, 100.5, 100.6, 100.2, 100.1]}
171        df = pd.DataFrame(dataset).set_index('date')
172        print(df)
173
174        # Adjust Bayesian Ridge
175        regr5 = linear_model.BayesianRidge()
176        obj_DeepMTS = ns.DeepMTS(regr5, lags = 1, n_hidden_features=5)
177        obj_DeepMTS.fit(df)
178        print(obj_DeepMTS.predict())
179
180        # with credible intervals
181        print(obj_DeepMTS.predict(return_std=True, level=80))
182
183        print(obj_DeepMTS.predict(return_std=True, level=95))
184        ```
185
186    """
187
188    # construct the object -----
189
190    def __init__(
191        self,
192        obj,
193        n_layers=3,
194        n_hidden_features=5,
195        activation_name="relu",
196        a=0.01,
197        nodes_sim="sobol",
198        bias=True,
199        dropout=0,
200        direct_link=True,
201        n_clusters=2,
202        cluster_encode=True,
203        type_clust="kmeans",
204        type_scaling=("std", "std", "std"),
205        lags=1,
206        type_pi="kde",
207        block_size=None,
208        replications=None,
209        kernel=None,
210        agg="mean",
211        seed=123,
212        backend="cpu",
213        verbose=0,
214        show_progress=True,
215    ):
216        assert int(lags) == lags, "parameter 'lags' should be an integer"
217        assert n_layers >= 1, "must have n_layers >= 1"
218        self.n_layers = int(n_layers)
219
220        self.obj = DeepRegressor(
221            obj=obj,
222            verbose=0,
223            n_layers=self.n_layers,
224            n_hidden_features=n_hidden_features,
225            activation_name=activation_name,
226            a=a,
227            nodes_sim=nodes_sim,
228            bias=bias,
229            dropout=dropout,
230            direct_link=direct_link,
231            n_clusters=n_clusters,
232            cluster_encode=cluster_encode,
233            type_clust=type_clust,
234            type_scaling=type_scaling,
235            seed=seed,
236            backend=backend,
237        )
238
239        super().__init__(
240            obj=self.obj,
241            n_hidden_features=n_hidden_features,
242            activation_name=activation_name,
243            a=a,
244            nodes_sim=nodes_sim,
245            bias=bias,
246            dropout=dropout,
247            direct_link=direct_link,
248            n_clusters=n_clusters,
249            cluster_encode=cluster_encode,
250            type_clust=type_clust,
251            type_scaling=type_scaling,
252            seed=seed,
253            type_pi=type_pi,
254            block_size=block_size,
255            replications=replications,
256            kernel=kernel,
257            agg=agg,
258            backend=backend,
259            verbose=verbose,
260            show_progress=show_progress,
261        )

Univariate and multivariate time series (DeepMTS) forecasting with Quasi-Randomized networks (Work in progress /!)

Parameters:

obj: object.
    any object containing a method fit (obj.fit()) and a method predict
    (obj.predict()).

n_layers: int.
    number of layers in the neural network.

n_hidden_features: int.
    number of nodes in the hidden layer.

activation_name: str.
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'.

a: float.
    hyperparameter for 'prelu' or 'elu' activation function.

nodes_sim: str.
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'.

bias: boolean.
    indicates if the hidden layer contains a bias term (True) or not
    (False).

dropout: float.
    regularization parameter; (random) percentage of nodes dropped out
    of the training.

direct_link: boolean.
    indicates if the original predictors are included (True) in model's fitting or not (False).

n_clusters: int.
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering).

cluster_encode: bool.
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding.

type_clust: str.
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm').

type_scaling: a tuple of 3 strings.
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax').

lags: int.
    number of lags used for each time series.

type_pi: str.
    type of prediction interval; currently:
    - "gaussian": simple, fast, but: assumes stationarity of Gaussian in-sample residuals and independence in the multivariate case
    - "kde": based on Kernel Density Estimation of in-sample residuals
    - "bootstrap": based on independent bootstrap of in-sample residuals
    - "block-bootstrap": based on basic block bootstrap of in-sample residuals
    - "scp-kde": Sequential split conformal prediction with Kernel Density Estimation of calibrated residuals
    - "scp-bootstrap": Sequential split conformal prediction with independent bootstrap of calibrated residuals
    - "scp-block-bootstrap": Sequential split conformal prediction with basic block bootstrap of calibrated residuals
    - "scp2-kde": Sequential split conformal prediction with Kernel Density Estimation of standardized calibrated residuals
    - "scp2-bootstrap": Sequential split conformal prediction with independent bootstrap of standardized calibrated residuals
    - "scp2-block-bootstrap": Sequential split conformal prediction with basic block bootstrap of standardized calibrated residuals

block_size: int.
    size of block for 'type_pi' in ("block-bootstrap", "scp-block-bootstrap", "scp2-block-bootstrap").
    Default is round(3.15*(n_residuals^1/3))

replications: int.
    number of replications (if needed, for predictive simulation). Default is 'None'.

kernel: str.
    the kernel to use for residuals density estimation (used for predictive simulation). Currently, either 'gaussian' or 'tophat'.

agg: str.
    either "mean" or "median" for simulation of bootstrap aggregating

seed: int.
    reproducibility seed for nodes_sim=='uniform' or predictive simulation.

backend: str.
    "cpu" or "gpu" or "tpu".

verbose: int.
    0: not printing; 1: printing

show_progress: bool.
    True: progress bar when fitting each series; False: no progress bar when fitting each series

Attributes:

fit_objs_: dict
    objects adjusted to each individual time series

y_: {array-like}
    DeepMTS responses (most recent observations first)

X_: {array-like}
    DeepMTS lags

xreg_: {array-like}
    external regressors

y_means_: dict
    a dictionary of each series mean values

preds_: {array-like}
    successive model predictions

preds_std_: {array-like}
    standard deviation around the predictions

return_std_: boolean
    return uncertainty or not (set in predict)

df_: data frame
    the input data frame, in case a data.frame is provided to `fit`

Examples:

Example 1:

import nnetsauce as ns
    import numpy as np
    from sklearn import linear_model
    np.random.seed(123)
 
M = np.random.rand(10, 3) M[:,0] = 10M[:,0] M[:,2] = 25M[:,2] print(M)
# Adjust Bayesian Ridge regr4 = linear_model.BayesianRidge() obj_DeepMTS = ns.DeepMTS(regr4, lags = 1, n_hidden_features=5) obj_DeepMTS.fit(M) print(obj_DeepMTS.predict())
# with credible intervals print(obj_DeepMTS.predict(return_std=True, level=80))
print(obj_DeepMTS.predict(return_std=True, level=95))

Example 2:

import nnetsauce as ns
    import numpy as np
    from sklearn import linear_model
 
dataset = { 'date' : ['2001-01-01', '2002-01-01', '2003-01-01', '2004-01-01', '2005-01-01'], 'series1' : [34, 30, 35.6, 33.3, 38.1], 'series2' : [4, 5.5, 5.6, 6.3, 5.1], 'series3' : [100, 100.5, 100.6, 100.2, 100.1]} df = pd.DataFrame(dataset).set_index('date') print(df)
# Adjust Bayesian Ridge regr5 = linear_model.BayesianRidge() obj_DeepMTS = ns.DeepMTS(regr5, lags = 1, n_hidden_features=5) obj_DeepMTS.fit(df) print(obj_DeepMTS.predict())
# with credible intervals print(obj_DeepMTS.predict(return_std=True, level=80))
print(obj_DeepMTS.predict(return_std=True, level=95))

class Downloader:
 6class Downloader:
 7    """Download datasets from data sources (R-universe for now)"""
 8
 9    def __init__(self):
10        self.pkgname = None
11        self.dataset = None
12        self.source = None
13        self.url = None
14        self.request = None
15
16    def download(
17        self,
18        pkgname="MASS",
19        dataset="Boston",
20        source="https://cran.r-universe.dev/",
21        **kwargs
22    ):
23        """Download datasets from data sources (R-universe for now)
24
25        Examples:
26
27        ```python
28        import nnetsauce as ns
29
30        downloader = ns.Downloader()
31        df = downloader.download(pkgname="MASS", dataset="Boston")
32        ```
33
34        """
35        self.pkgname = pkgname
36        self.dataset = dataset
37        self.source = source
38        self.url = source + pkgname + "/data/" + dataset + "/json"
39        self.request = requests.get(self.url)
40        return pd.DataFrame(self.request.json(), **kwargs)

Download datasets from data sources (R-universe for now)

def download( self, pkgname='MASS', dataset='Boston', source='https://cran.r-universe.dev/', **kwargs):
16    def download(
17        self,
18        pkgname="MASS",
19        dataset="Boston",
20        source="https://cran.r-universe.dev/",
21        **kwargs
22    ):
23        """Download datasets from data sources (R-universe for now)
24
25        Examples:
26
27        ```python
28        import nnetsauce as ns
29
30        downloader = ns.Downloader()
31        df = downloader.download(pkgname="MASS", dataset="Boston")
32        ```
33
34        """
35        self.pkgname = pkgname
36        self.dataset = dataset
37        self.source = source
38        self.url = source + pkgname + "/data/" + dataset + "/json"
39        self.request = requests.get(self.url)
40        return pd.DataFrame(self.request.json(), **kwargs)

Download datasets from data sources (R-universe for now)

Examples:

import nnetsauce as ns

downloader = ns.Downloader()
df = downloader.download(pkgname="MASS", dataset="Boston")
class GLMClassifier(nnetsauce.glm.glm.GLM, sklearn.base.ClassifierMixin):
 16class GLMClassifier(GLM, ClassifierMixin):
 17    """Generalized 'linear' models using quasi-randomized networks (classification)
 18
 19    Parameters:
 20
 21        n_hidden_features: int
 22            number of nodes in the hidden layer
 23
 24        lambda1: float
 25            regularization parameter for GLM coefficients on original features
 26
 27        alpha1: float
 28            controls compromize between l1 and l2 norm of GLM coefficients on original features
 29
 30        lambda2: float
 31            regularization parameter for GLM coefficients on nonlinear features
 32
 33        alpha2: float
 34            controls compromize between l1 and l2 norm of GLM coefficients on nonlinear features
 35
 36        activation_name: str
 37            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 38
 39        a: float
 40            hyperparameter for 'prelu' or 'elu' activation function
 41
 42        nodes_sim: str
 43            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 44            'uniform'
 45
 46        bias: boolean
 47            indicates if the hidden layer contains a bias term (True) or not
 48            (False)
 49
 50        dropout: float
 51            regularization parameter; (random) percentage of nodes dropped out
 52            of the training
 53
 54        direct_link: boolean
 55            indicates if the original predictors are included (True) in model's
 56            fitting or not (False)
 57
 58        n_clusters: int
 59            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 60                no clustering)
 61
 62        cluster_encode: bool
 63            defines how the variable containing clusters is treated (default is one-hot)
 64            if `False`, then labels are used, without one-hot encoding
 65
 66        type_clust: str
 67            type of clustering method: currently k-means ('kmeans') or Gaussian
 68            Mixture Model ('gmm')
 69
 70        type_scaling: a tuple of 3 strings
 71            scaling methods for inputs, hidden layer, and clustering respectively
 72            (and when relevant).
 73            Currently available: standardization ('std') or MinMax scaling ('minmax')
 74
 75        optimizer: object
 76            optimizer, from class nnetsauce.Optimizer
 77
 78        seed: int
 79            reproducibility seed for nodes_sim=='uniform'
 80
 81    Attributes:
 82
 83        beta_: vector
 84            regression coefficients
 85
 86    Examples:
 87
 88    See [https://github.com/Techtonique/nnetsauce/blob/master/examples/glm_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/glm_classification.py)
 89
 90    """
 91
 92    # construct the object -----
 93
 94    def __init__(
 95        self,
 96        n_hidden_features=5,
 97        lambda1=0.01,
 98        alpha1=0.5,
 99        lambda2=0.01,
100        alpha2=0.5,
101        family="expit",
102        activation_name="relu",
103        a=0.01,
104        nodes_sim="sobol",
105        bias=True,
106        dropout=0,
107        direct_link=True,
108        n_clusters=2,
109        cluster_encode=True,
110        type_clust="kmeans",
111        type_scaling=("std", "std", "std"),
112        optimizer=Optimizer(),
113        seed=123,
114    ):
115        super().__init__(
116            n_hidden_features=n_hidden_features,
117            lambda1=lambda1,
118            alpha1=alpha1,
119            lambda2=lambda2,
120            alpha2=alpha2,
121            activation_name=activation_name,
122            a=a,
123            nodes_sim=nodes_sim,
124            bias=bias,
125            dropout=dropout,
126            direct_link=direct_link,
127            n_clusters=n_clusters,
128            cluster_encode=cluster_encode,
129            type_clust=type_clust,
130            type_scaling=type_scaling,
131            optimizer=optimizer,
132            seed=seed,
133        )
134
135        self.family = family
136
137    def logit_loss(self, Y, row_index, XB):
138        self.n_classes = Y.shape[1]  # len(np.unique(y))
139        # Y = mo.one_hot_encode2(y, self.n_classes)
140        # Y = self.optimizer.one_hot_encode(y, self.n_classes)
141
142        # max_double = 709.0 # only if softmax
143        # XB[XB > max_double] = max_double
144        XB[XB > 709.0] = 709.0
145
146        if row_index is None:
147            return -np.mean(np.sum(Y * XB, axis=1) - logsumexp(XB))
148
149        return -np.mean(np.sum(Y[row_index, :] * XB, axis=1) - logsumexp(XB))
150
151    def expit_erf_loss(self, Y, row_index, XB):
152        # self.n_classes = len(np.unique(y))
153        # Y = mo.one_hot_encode2(y, self.n_classes)
154        # Y = self.optimizer.one_hot_encode(y, self.n_classes)
155        self.n_classes = Y.shape[1]
156
157        if row_index is None:
158            return -np.mean(np.sum(Y * XB, axis=1) - logsumexp(XB))
159
160        return -np.mean(np.sum(Y[row_index, :] * XB, axis=1) - logsumexp(XB))
161
162    def loss_func(
163        self,
164        beta,
165        group_index,
166        X,
167        Y,
168        y,
169        row_index=None,
170        type_loss="logit",
171        **kwargs
172    ):
173        res = {
174            "logit": self.logit_loss,
175            "expit": self.expit_erf_loss,
176            "erf": self.expit_erf_loss,
177        }
178
179        if row_index is None:
180            row_index = range(len(y))
181            XB = self.compute_XB(
182                X,
183                beta=np.reshape(beta, (X.shape[1], self.n_classes), order="F"),
184            )
185
186            return res[type_loss](Y, row_index, XB) + self.compute_penalty(
187                group_index=group_index, beta=beta
188            )
189
190        XB = self.compute_XB(
191            X,
192            beta=np.reshape(beta, (X.shape[1], self.n_classes), order="F"),
193            row_index=row_index,
194        )
195
196        return res[type_loss](Y, row_index, XB) + self.compute_penalty(
197            group_index=group_index, beta=beta
198        )
199
200    def fit(self, X, y, **kwargs):
201        """Fit GLM model to training data (X, y).
202
203        Args:
204
205            X: {array-like}, shape = [n_samples, n_features]
206                Training vectors, where n_samples is the number
207                of samples and n_features is the number of features.
208
209            y: array-like, shape = [n_samples]
210                Target values.
211
212            **kwargs: additional parameters to be passed to
213                    self.cook_training_set or self.obj.fit
214
215        Returns:
216
217            self: object
218
219        """
220
221        assert mx.is_factor(
222            y
223        ), "y must contain only integers"  # change is_factor and subsampling everywhere
224
225        self.classes_ = np.unique(y)  # for compatibility with sklearn
226        self.n_classes_ = len(self.classes_)  # for compatibility with sklearn
227
228        self.beta_ = None
229
230        n, p = X.shape
231
232        self.group_index = n * X.shape[1]
233
234        self.n_classes = len(np.unique(y))
235
236        output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
237
238        # Y = mo.one_hot_encode2(output_y, self.n_classes)
239        Y = self.optimizer.one_hot_encode(output_y, self.n_classes)
240
241        # initialization
242        beta_ = np.linalg.lstsq(scaled_Z, Y, rcond=None)[0]
243
244        # optimization
245        # fit(self, loss_func, response, x0, **kwargs):
246        # loss_func(self, beta, group_index, X, y,
247        #          row_index=None, type_loss="gaussian",
248        #          **kwargs)
249        self.optimizer.fit(
250            self.loss_func,
251            response=y,
252            x0=beta_.flatten(order="F"),
253            group_index=self.group_index,
254            X=scaled_Z,
255            Y=Y,
256            y=y,
257            type_loss=self.family,
258        )
259
260        self.beta_ = self.optimizer.results[0]
261        self.classes_ = np.unique(y)
262
263        return self
264
265    def predict(self, X, **kwargs):
266        """Predict test data X.
267
268        Args:
269
270            X: {array-like}, shape = [n_samples, n_features]
271                Training vectors, where n_samples is the number
272                of samples and n_features is the number of features.
273
274            **kwargs: additional parameters to be passed to
275                    self.cook_test_set
276
277        Returns:
278
279            model predictions: {array-like}
280
281        """
282
283        return np.argmax(self.predict_proba(X, **kwargs), axis=1)
284
285    def predict_proba(self, X, **kwargs):
286        """Predict probabilities for test data X.
287
288        Args:
289
290            X: {array-like}, shape = [n_samples, n_features]
291                Training vectors, where n_samples is the number
292                of samples and n_features is the number of features.
293
294            **kwargs: additional parameters to be passed to
295                    self.cook_test_set
296
297        Returns:
298
299            probability estimates for test data: {array-like}
300
301        """
302        if len(X.shape) == 1:
303            n_features = X.shape[0]
304            new_X = mo.rbind(
305                X.reshape(1, n_features),
306                np.ones(n_features).reshape(1, n_features),
307            )
308
309            Z = self.cook_test_set(new_X, **kwargs)
310
311        else:
312            Z = self.cook_test_set(X, **kwargs)
313
314        ZB = mo.safe_sparse_dot(
315            Z,
316            self.beta_.reshape(
317                self.n_classes,
318                X.shape[1] + self.n_hidden_features + self.n_clusters,
319            ).T,
320        )
321
322        if self.family == "logit":
323            exp_ZB = np.exp(ZB)
324
325            return exp_ZB / exp_ZB.sum(axis=1)[:, None]
326
327        if self.family == "expit":
328            exp_ZB = expit(ZB)
329
330            return exp_ZB / exp_ZB.sum(axis=1)[:, None]
331
332        if self.family == "erf":
333            exp_ZB = 0.5 * (1 + erf(ZB))
334
335            return exp_ZB / exp_ZB.sum(axis=1)[:, None]

Generalized 'linear' models using quasi-randomized networks (classification)

Parameters:

n_hidden_features: int
    number of nodes in the hidden layer

lambda1: float
    regularization parameter for GLM coefficients on original features

alpha1: float
    controls compromize between l1 and l2 norm of GLM coefficients on original features

lambda2: float
    regularization parameter for GLM coefficients on nonlinear features

alpha2: float
    controls compromize between l1 and l2 norm of GLM coefficients on nonlinear features

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original predictors are included (True) in model's
    fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

optimizer: object
    optimizer, from class nnetsauce.Optimizer

seed: int
    reproducibility seed for nodes_sim=='uniform'

Attributes:

beta_: vector
    regression coefficients

Examples:

See https://github.com/Techtonique/nnetsauce/blob/master/examples/glm_classification.py

def fit(self, X, y, **kwargs):
200    def fit(self, X, y, **kwargs):
201        """Fit GLM model to training data (X, y).
202
203        Args:
204
205            X: {array-like}, shape = [n_samples, n_features]
206                Training vectors, where n_samples is the number
207                of samples and n_features is the number of features.
208
209            y: array-like, shape = [n_samples]
210                Target values.
211
212            **kwargs: additional parameters to be passed to
213                    self.cook_training_set or self.obj.fit
214
215        Returns:
216
217            self: object
218
219        """
220
221        assert mx.is_factor(
222            y
223        ), "y must contain only integers"  # change is_factor and subsampling everywhere
224
225        self.classes_ = np.unique(y)  # for compatibility with sklearn
226        self.n_classes_ = len(self.classes_)  # for compatibility with sklearn
227
228        self.beta_ = None
229
230        n, p = X.shape
231
232        self.group_index = n * X.shape[1]
233
234        self.n_classes = len(np.unique(y))
235
236        output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
237
238        # Y = mo.one_hot_encode2(output_y, self.n_classes)
239        Y = self.optimizer.one_hot_encode(output_y, self.n_classes)
240
241        # initialization
242        beta_ = np.linalg.lstsq(scaled_Z, Y, rcond=None)[0]
243
244        # optimization
245        # fit(self, loss_func, response, x0, **kwargs):
246        # loss_func(self, beta, group_index, X, y,
247        #          row_index=None, type_loss="gaussian",
248        #          **kwargs)
249        self.optimizer.fit(
250            self.loss_func,
251            response=y,
252            x0=beta_.flatten(order="F"),
253            group_index=self.group_index,
254            X=scaled_Z,
255            Y=Y,
256            y=y,
257            type_loss=self.family,
258        )
259
260        self.beta_ = self.optimizer.results[0]
261        self.classes_ = np.unique(y)
262
263        return self

Fit GLM model to training data (X, y).

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, **kwargs):
265    def predict(self, X, **kwargs):
266        """Predict test data X.
267
268        Args:
269
270            X: {array-like}, shape = [n_samples, n_features]
271                Training vectors, where n_samples is the number
272                of samples and n_features is the number of features.
273
274            **kwargs: additional parameters to be passed to
275                    self.cook_test_set
276
277        Returns:
278
279            model predictions: {array-like}
280
281        """
282
283        return np.argmax(self.predict_proba(X, **kwargs), axis=1)

Predict test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
def predict_proba(self, X, **kwargs):
285    def predict_proba(self, X, **kwargs):
286        """Predict probabilities for test data X.
287
288        Args:
289
290            X: {array-like}, shape = [n_samples, n_features]
291                Training vectors, where n_samples is the number
292                of samples and n_features is the number of features.
293
294            **kwargs: additional parameters to be passed to
295                    self.cook_test_set
296
297        Returns:
298
299            probability estimates for test data: {array-like}
300
301        """
302        if len(X.shape) == 1:
303            n_features = X.shape[0]
304            new_X = mo.rbind(
305                X.reshape(1, n_features),
306                np.ones(n_features).reshape(1, n_features),
307            )
308
309            Z = self.cook_test_set(new_X, **kwargs)
310
311        else:
312            Z = self.cook_test_set(X, **kwargs)
313
314        ZB = mo.safe_sparse_dot(
315            Z,
316            self.beta_.reshape(
317                self.n_classes,
318                X.shape[1] + self.n_hidden_features + self.n_clusters,
319            ).T,
320        )
321
322        if self.family == "logit":
323            exp_ZB = np.exp(ZB)
324
325            return exp_ZB / exp_ZB.sum(axis=1)[:, None]
326
327        if self.family == "expit":
328            exp_ZB = expit(ZB)
329
330            return exp_ZB / exp_ZB.sum(axis=1)[:, None]
331
332        if self.family == "erf":
333            exp_ZB = 0.5 * (1 + erf(ZB))
334
335            return exp_ZB / exp_ZB.sum(axis=1)[:, None]

Predict probabilities for test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

probability estimates for test data: {array-like}
class GLMRegressor(nnetsauce.glm.glm.GLM, sklearn.base.RegressorMixin):
 14class GLMRegressor(GLM, RegressorMixin):
 15    """Generalized 'linear' models using quasi-randomized networks (regression)
 16
 17    Attributes:
 18
 19        n_hidden_features: int
 20            number of nodes in the hidden layer
 21
 22        lambda1: float
 23            regularization parameter for GLM coefficients on original features
 24
 25        alpha1: float
 26            controls compromize between l1 and l2 norm of GLM coefficients on original features
 27
 28        lambda2: float
 29            regularization parameter for GLM coefficients on nonlinear features
 30
 31        alpha2: float
 32            controls compromize between l1 and l2 norm of GLM coefficients on nonlinear features
 33
 34        family: str
 35            "gaussian", "laplace" or "poisson" (for now)
 36
 37        activation_name: str
 38            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 39
 40        a: float
 41            hyperparameter for 'prelu' or 'elu' activation function
 42
 43        nodes_sim: str
 44            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 45            'uniform'
 46
 47        bias: boolean
 48            indicates if the hidden layer contains a bias term (True) or not
 49            (False)
 50
 51        dropout: float
 52            regularization parameter; (random) percentage of nodes dropped out
 53            of the training
 54
 55        direct_link: boolean
 56            indicates if the original predictors are included (True) in model's
 57            fitting or not (False)
 58
 59        n_clusters: int
 60            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 61                no clustering)
 62
 63        cluster_encode: bool
 64            defines how the variable containing clusters is treated (default is one-hot)
 65            if `False`, then labels are used, without one-hot encoding
 66
 67        type_clust: str
 68            type of clustering method: currently k-means ('kmeans') or Gaussian
 69            Mixture Model ('gmm')
 70
 71        type_scaling: a tuple of 3 strings
 72            scaling methods for inputs, hidden layer, and clustering respectively
 73            (and when relevant).
 74            Currently available: standardization ('std') or MinMax scaling ('minmax')
 75
 76        optimizer: object
 77            optimizer, from class nnetsauce.utils.Optimizer
 78
 79        seed: int
 80            reproducibility seed for nodes_sim=='uniform'
 81
 82    Attributes:
 83
 84        beta_: vector
 85            regression coefficients
 86
 87    Examples:
 88
 89    See [https://github.com/Techtonique/nnetsauce/blob/master/examples/glm_regression.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/glm_regression.py)
 90
 91    """
 92
 93    # construct the object -----
 94
 95    def __init__(
 96        self,
 97        n_hidden_features=5,
 98        lambda1=0.01,
 99        alpha1=0.5,
100        lambda2=0.01,
101        alpha2=0.5,
102        family="gaussian",
103        activation_name="relu",
104        a=0.01,
105        nodes_sim="sobol",
106        bias=True,
107        dropout=0,
108        direct_link=True,
109        n_clusters=2,
110        cluster_encode=True,
111        type_clust="kmeans",
112        type_scaling=("std", "std", "std"),
113        optimizer=Optimizer(),
114        seed=123,
115    ):
116        super().__init__(
117            n_hidden_features=n_hidden_features,
118            lambda1=lambda1,
119            alpha1=alpha1,
120            lambda2=lambda2,
121            alpha2=alpha2,
122            activation_name=activation_name,
123            a=a,
124            nodes_sim=nodes_sim,
125            bias=bias,
126            dropout=dropout,
127            direct_link=direct_link,
128            n_clusters=n_clusters,
129            cluster_encode=cluster_encode,
130            type_clust=type_clust,
131            type_scaling=type_scaling,
132            optimizer=optimizer,
133            seed=seed,
134        )
135
136        self.family = family
137
138    def gaussian_loss(self, y, row_index, XB):
139        return 0.5 * np.mean(np.square(y[row_index] - XB))
140
141    def laplace_loss(self, y, row_index, XB):
142        return 0.5 * np.mean(np.abs(y[row_index] - XB))
143
144    def poisson_loss(self, y, row_index, XB):
145        return -np.mean(y[row_index] * XB - np.exp(XB))
146
147    def loss_func(
148        self,
149        beta,
150        group_index,
151        X,
152        y,
153        row_index=None,
154        type_loss="gaussian",
155        **kwargs
156    ):
157        res = {
158            "gaussian": self.gaussian_loss,
159            "laplace": self.laplace_loss,
160            "poisson": self.poisson_loss,
161        }
162
163        if row_index is None:
164            row_index = range(len(y))
165            XB = self.compute_XB(X, beta=beta)
166
167            return res[type_loss](y, row_index, XB) + self.compute_penalty(
168                group_index=group_index, beta=beta
169            )
170
171        XB = self.compute_XB(X, beta=beta, row_index=row_index)
172
173        return res[type_loss](y, row_index, XB) + self.compute_penalty(
174            group_index=group_index, beta=beta
175        )
176
177    def fit(self, X, y, **kwargs):
178        """Fit GLM model to training data (X, y).
179
180        Args:
181
182            X: {array-like}, shape = [n_samples, n_features]
183                Training vectors, where n_samples is the number
184                of samples and n_features is the number of features.
185
186            y: array-like, shape = [n_samples]
187                Target values.
188
189            **kwargs: additional parameters to be passed to
190                    self.cook_training_set or self.obj.fit
191
192        Returns:
193
194            self: object
195
196        """
197
198        self.beta_ = None
199
200        self.n_iter = 0
201
202        n, self.group_index = X.shape
203
204        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
205
206        n_Z = scaled_Z.shape[0]
207
208        # initialization
209        beta_ = np.linalg.lstsq(scaled_Z, centered_y, rcond=None)[0]
210
211        # optimization
212        # fit(self, loss_func, response, x0, **kwargs):
213        # loss_func(self, beta, group_index, X, y,
214        #          row_index=None, type_loss="gaussian",
215        #          **kwargs)
216        self.optimizer.fit(
217            self.loss_func,
218            response=centered_y,
219            x0=beta_,
220            group_index=self.group_index,
221            X=scaled_Z,
222            y=centered_y,
223            type_loss=self.family,
224            **kwargs
225        )
226
227        self.beta_ = self.optimizer.results[0]
228
229        return self
230
231    def predict(self, X, **kwargs):
232        """Predict test data X.
233
234        Args:
235
236            X: {array-like}, shape = [n_samples, n_features]
237                Training vectors, where n_samples is the number
238                of samples and n_features is the number of features.
239
240            **kwargs: additional parameters to be passed to
241                    self.cook_test_set
242
243        Returns:
244
245            model predictions: {array-like}
246
247        """
248
249        if len(X.shape) == 1:
250            n_features = X.shape[0]
251            new_X = mo.rbind(
252                X.reshape(1, n_features),
253                np.ones(n_features).reshape(1, n_features),
254            )
255
256            return (
257                self.y_mean_
258                + np.dot(self.cook_test_set(new_X, **kwargs), self.beta_)
259            )[0]
260
261        return self.y_mean_ + np.dot(
262            self.cook_test_set(X, **kwargs), self.beta_
263        )

Generalized 'linear' models using quasi-randomized networks (regression)

Attributes:

n_hidden_features: int
    number of nodes in the hidden layer

lambda1: float
    regularization parameter for GLM coefficients on original features

alpha1: float
    controls compromize between l1 and l2 norm of GLM coefficients on original features

lambda2: float
    regularization parameter for GLM coefficients on nonlinear features

alpha2: float
    controls compromize between l1 and l2 norm of GLM coefficients on nonlinear features

family: str
    "gaussian", "laplace" or "poisson" (for now)

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original predictors are included (True) in model's
    fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

optimizer: object
    optimizer, from class nnetsauce.utils.Optimizer

seed: int
    reproducibility seed for nodes_sim=='uniform'

Attributes:

beta_: vector
    regression coefficients

Examples:

See https://github.com/Techtonique/nnetsauce/blob/master/examples/glm_regression.py

def fit(self, X, y, **kwargs):
177    def fit(self, X, y, **kwargs):
178        """Fit GLM model to training data (X, y).
179
180        Args:
181
182            X: {array-like}, shape = [n_samples, n_features]
183                Training vectors, where n_samples is the number
184                of samples and n_features is the number of features.
185
186            y: array-like, shape = [n_samples]
187                Target values.
188
189            **kwargs: additional parameters to be passed to
190                    self.cook_training_set or self.obj.fit
191
192        Returns:
193
194            self: object
195
196        """
197
198        self.beta_ = None
199
200        self.n_iter = 0
201
202        n, self.group_index = X.shape
203
204        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
205
206        n_Z = scaled_Z.shape[0]
207
208        # initialization
209        beta_ = np.linalg.lstsq(scaled_Z, centered_y, rcond=None)[0]
210
211        # optimization
212        # fit(self, loss_func, response, x0, **kwargs):
213        # loss_func(self, beta, group_index, X, y,
214        #          row_index=None, type_loss="gaussian",
215        #          **kwargs)
216        self.optimizer.fit(
217            self.loss_func,
218            response=centered_y,
219            x0=beta_,
220            group_index=self.group_index,
221            X=scaled_Z,
222            y=centered_y,
223            type_loss=self.family,
224            **kwargs
225        )
226
227        self.beta_ = self.optimizer.results[0]
228
229        return self

Fit GLM model to training data (X, y).

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, **kwargs):
231    def predict(self, X, **kwargs):
232        """Predict test data X.
233
234        Args:
235
236            X: {array-like}, shape = [n_samples, n_features]
237                Training vectors, where n_samples is the number
238                of samples and n_features is the number of features.
239
240            **kwargs: additional parameters to be passed to
241                    self.cook_test_set
242
243        Returns:
244
245            model predictions: {array-like}
246
247        """
248
249        if len(X.shape) == 1:
250            n_features = X.shape[0]
251            new_X = mo.rbind(
252                X.reshape(1, n_features),
253                np.ones(n_features).reshape(1, n_features),
254            )
255
256            return (
257                self.y_mean_
258                + np.dot(self.cook_test_set(new_X, **kwargs), self.beta_)
259            )[0]
260
261        return self.y_mean_ + np.dot(
262            self.cook_test_set(X, **kwargs), self.beta_
263        )

Predict test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
class LazyClassifier(nnetsauce.custom.custom.Custom, sklearn.base.ClassifierMixin):
 84class LazyClassifier(Custom, ClassifierMixin):
 85    """
 86
 87        Fitting -- almost -- all the classification algorithms with nnetsauce's
 88        CustomClassifier and returning their scores.
 89
 90    Parameters:
 91
 92        verbose: int, optional (default=0)
 93            Any positive number for verbosity.
 94
 95        ignore_warnings: bool, optional (default=True)
 96            When set to True, warnings related to algorithms that were not
 97            run are ignored.
 98
 99        custom_metric: function, optional (default=None)
100            When function is provided, models are evaluated based on the
101            custom evaluation metric provided.
102
103        predictions: bool, optional (default=False)
104            When set to True, the predictions of all the models models are
105            returned as data frame.
106
107        sort_by: string, optional (default='Accuracy')
108            Sort models by a metric. Available options are 'Accuracy',
109            'Balanced Accuracy', 'ROC AUC', 'F1 Score' or a custom metric
110            identified by its name and provided by custom_metric.
111
112        random_state: int, optional (default=42)
113            Reproducibiility seed.
114
115        estimators: list, optional (default='all')
116            list of Estimators names or just 'all' for > 90
117            classifiers (default='all')
118
119        preprocess: bool, preprocessing is done when set to True
120
121        n_jobs: int, when possible, run in parallel
122            For now, only used by individual models that support it.
123
124        All the other parameters are the same as CustomClassifier's.
125
126    Examples:
127
128        ```python
129        import nnetsauce as ns
130        from sklearn.datasets import load_breast_cancer
131        from sklearn.model_selection import train_test_split
132        data = load_breast_cancer()
133        X = data.data
134        y= data.target
135        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2,
136        random_state=123)
137        clf = ns.LazyClassifier(verbose=0, ignore_warnings=True)
138        models, predictions = clf.fit(X_train, X_test, y_train, y_test)
139        model_dictionary = clf.provide_models(X_train,X_test,y_train,y_test)
140        print(models)
141        ```
142
143    """
144
145    def __init__(
146        self,
147        verbose=0,
148        ignore_warnings=True,
149        custom_metric=None,
150        predictions=False,
151        sort_by="Accuracy",
152        random_state=42,
153        estimators="all",
154        preprocess=False,
155        n_jobs=None,
156        # CustomClassifier attributes
157        obj=None,
158        n_hidden_features=5,
159        activation_name="relu",
160        a=0.01,
161        nodes_sim="sobol",
162        bias=True,
163        dropout=0,
164        direct_link=True,
165        n_clusters=2,
166        cluster_encode=True,
167        type_clust="kmeans",
168        type_scaling=("std", "std", "std"),
169        col_sample=1,
170        row_sample=1,
171        seed=123,
172        backend="cpu",
173    ):
174        self.verbose = verbose
175        self.ignore_warnings = ignore_warnings
176        self.custom_metric = custom_metric
177        self.predictions = predictions
178        self.sort_by = sort_by
179        self.models = {}
180        self.random_state = random_state
181        self.estimators = estimators
182        self.preprocess = preprocess
183        self.n_jobs = n_jobs
184        super().__init__(
185            obj=obj,
186            n_hidden_features=n_hidden_features,
187            activation_name=activation_name,
188            a=a,
189            nodes_sim=nodes_sim,
190            bias=bias,
191            dropout=dropout,
192            direct_link=direct_link,
193            n_clusters=n_clusters,
194            cluster_encode=cluster_encode,
195            type_clust=type_clust,
196            type_scaling=type_scaling,
197            col_sample=col_sample,
198            row_sample=row_sample,
199            seed=seed,
200            backend=backend,
201        )
202
203    def fit(self, X_train, X_test, y_train, y_test):
204        """Fit classifiers to X_train and y_train, predict and score on X_test,
205        y_test.
206
207        Parameters:
208
209            X_train: array-like,
210                Training vectors, where rows is the number of samples
211                and columns is the number of features.
212
213            X_test: array-like,
214                Testing vectors, where rows is the number of samples
215                and columns is the number of features.
216
217            y_train: array-like,
218                Training vectors, where rows is the number of samples
219                and columns is the number of features.
220
221            y_test: array-like,
222                Testing vectors, where rows is the number of samples
223                and columns is the number of features.
224
225        Returns:
226
227            scores: Pandas DataFrame
228                Returns metrics of all the models in a Pandas DataFrame.
229
230            predictions: Pandas DataFrame
231                Returns predictions of all the models in a Pandas DataFrame.
232        """
233        Accuracy = []
234        B_Accuracy = []
235        ROC_AUC = []
236        F1 = []
237        names = []
238        TIME = []
239        predictions = {}
240
241        if self.custom_metric is not None:
242            CUSTOM_METRIC = []
243
244        if isinstance(X_train, np.ndarray):
245            X_train = pd.DataFrame(X_train)
246            X_test = pd.DataFrame(X_test)
247
248        numeric_features = X_train.select_dtypes(include=[np.number]).columns
249        categorical_features = X_train.select_dtypes(include=["object"]).columns
250
251        categorical_low, categorical_high = get_card_split(
252            X_train, categorical_features
253        )
254
255        if self.preprocess is True:
256
257            preprocessor = ColumnTransformer(
258                transformers=[
259                    ("numeric", numeric_transformer, numeric_features),
260                    (
261                        "categorical_low",
262                        categorical_transformer_low,
263                        categorical_low,
264                    ),
265                    (
266                        "categorical_high",
267                        categorical_transformer_high,
268                        categorical_high,
269                    ),
270                ]
271            )
272
273        if self.estimators == "all":
274
275            self.classifiers = (
276                CLASSIFIERS + MULTITASKCLASSIFIERS + SIMPLEMULTITASKCLASSIFIERS
277            )
278
279        else:  # list custom estimators, by their names
280
281            self.classifiers = (
282                [
283                    ("CustomClassifier(" + est[0] + ")", est[1])
284                    for est in all_estimators()
285                    if (
286                        issubclass(est[1], ClassifierMixin)
287                        and (est[0] in self.estimators)
288                    )
289                ]
290                + [
291                    (
292                        "MultitaskClassifier(" + est[0] + ")",
293                        partial(MultitaskClassifier, obj=est[1]()),
294                    )
295                    for est in all_estimators()
296                    if (
297                        issubclass(est[1], RegressorMixin)
298                        and (est[0] in self.estimators)
299                    )
300                ]
301                + [
302                    (
303                        "SimpleMultitaskClassifier(" + est[0] + ")",
304                        partial(SimpleMultitaskClassifier, obj=est[1]()),
305                    )
306                    for est in all_estimators()
307                    if (
308                        issubclass(est[1], RegressorMixin)
309                        and (est[0] in self.estimators)
310                    )
311                ]
312            )
313
314        if self.preprocess is True:
315
316            for name, model in tqdm(self.classifiers):  # do parallel exec
317
318                other_args = (
319                    {}
320                )  # use this trick for `random_state` too --> refactor
321                try:
322                    if (
323                        "n_jobs" in model().get_params().keys()
324                        and name.find("LogisticRegression") == -1
325                    ):
326                        other_args["n_jobs"] = self.n_jobs
327                except Exception:
328                    pass
329
330                start = time.time()
331
332                try:
333                    if "random_state" in model().get_params().keys():
334                        pipe = Pipeline(
335                            [
336                                ("preprocessor", preprocessor),
337                                (
338                                    "classifier",
339                                    CustomClassifier(
340                                        obj=model(
341                                            random_state=self.random_state,
342                                            **other_args
343                                        ),
344                                        n_hidden_features=self.n_hidden_features,
345                                        activation_name=self.activation_name,
346                                        a=self.a,
347                                        nodes_sim=self.nodes_sim,
348                                        bias=self.bias,
349                                        dropout=self.dropout,
350                                        direct_link=self.direct_link,
351                                        n_clusters=self.n_clusters,
352                                        cluster_encode=self.cluster_encode,
353                                        type_clust=self.type_clust,
354                                        type_scaling=self.type_scaling,
355                                        col_sample=self.col_sample,
356                                        row_sample=self.row_sample,
357                                        seed=self.seed,
358                                        backend=self.backend,
359                                    ),
360                                ),
361                            ]
362                        )
363                    else:
364                        pipe = Pipeline(
365                            [
366                                ("preprocessor", preprocessor),
367                                (
368                                    "classifier",
369                                    CustomClassifier(
370                                        obj=model(**other_args),
371                                        n_hidden_features=self.n_hidden_features,
372                                        activation_name=self.activation_name,
373                                        a=self.a,
374                                        nodes_sim=self.nodes_sim,
375                                        bias=self.bias,
376                                        dropout=self.dropout,
377                                        direct_link=self.direct_link,
378                                        n_clusters=self.n_clusters,
379                                        cluster_encode=self.cluster_encode,
380                                        type_clust=self.type_clust,
381                                        type_scaling=self.type_scaling,
382                                        col_sample=self.col_sample,
383                                        row_sample=self.row_sample,
384                                        seed=self.seed,
385                                        backend=self.backend,
386                                    ),
387                                ),
388                            ]
389                        )
390
391                    pipe.fit(X_train, y_train)
392                    self.models[name] = pipe
393                    y_pred = pipe.predict(X_test)
394                    accuracy = accuracy_score(y_test, y_pred, normalize=True)
395                    b_accuracy = balanced_accuracy_score(y_test, y_pred)
396                    f1 = f1_score(y_test, y_pred, average="weighted")
397                    try:
398                        roc_auc = roc_auc_score(y_test, y_pred)
399                    except Exception as exception:
400                        roc_auc = None
401                        if self.ignore_warnings is False:
402                            print("ROC AUC couldn't be calculated for " + name)
403                            print(exception)
404                    names.append(name)
405                    Accuracy.append(accuracy)
406                    B_Accuracy.append(b_accuracy)
407                    ROC_AUC.append(roc_auc)
408                    F1.append(f1)
409                    TIME.append(time.time() - start)
410                    if self.custom_metric is not None:
411                        custom_metric = self.custom_metric(y_test, y_pred)
412                        CUSTOM_METRIC.append(custom_metric)
413                    if self.verbose > 0:
414                        if self.custom_metric is not None:
415                            print(
416                                {
417                                    "Model": name,
418                                    "Accuracy": accuracy,
419                                    "Balanced Accuracy": b_accuracy,
420                                    "ROC AUC": roc_auc,
421                                    "F1 Score": f1,
422                                    self.custom_metric.__name__: custom_metric,
423                                    "Time taken": time.time() - start,
424                                }
425                            )
426                        else:
427                            print(
428                                {
429                                    "Model": name,
430                                    "Accuracy": accuracy,
431                                    "Balanced Accuracy": b_accuracy,
432                                    "ROC AUC": roc_auc,
433                                    "F1 Score": f1,
434                                    "Time taken": time.time() - start,
435                                }
436                            )
437                    if self.predictions:
438                        predictions[name] = y_pred
439                except Exception as exception:
440                    try:
441                        if self.ignore_warnings is False:
442                            print(name + " model failed to execute")
443                            print(exception)
444                    except Exception as exception:
445                        pass
446
447        else:  # if self.preprocess is False:
448
449            for name, model in tqdm(self.classifiers):  # do parallel exec
450                other_args = (
451                    {}
452                )  # use this trick for `random_state` too --> refactor
453                try:
454                    if (
455                        "n_jobs" in model().get_params().keys()
456                        and name.find("LogisticRegression") == -1
457                    ):
458                        other_args["n_jobs"] = self.n_jobs
459                except Exception:
460                    pass
461
462                start = time.time()
463                try:
464                    if "random_state" in model().get_params().keys():
465                        pipe = CustomClassifier(
466                            obj=model(
467                                random_state=self.random_state, **other_args
468                            ),
469                            n_hidden_features=self.n_hidden_features,
470                            activation_name=self.activation_name,
471                            a=self.a,
472                            nodes_sim=self.nodes_sim,
473                            bias=self.bias,
474                            dropout=self.dropout,
475                            direct_link=self.direct_link,
476                            n_clusters=self.n_clusters,
477                            cluster_encode=self.cluster_encode,
478                            type_clust=self.type_clust,
479                            type_scaling=self.type_scaling,
480                            col_sample=self.col_sample,
481                            row_sample=self.row_sample,
482                            seed=self.seed,
483                            backend=self.backend,
484                        )
485                    else:
486                        pipe = CustomClassifier(
487                            obj=model(**other_args),
488                            n_hidden_features=self.n_hidden_features,
489                            activation_name=self.activation_name,
490                            a=self.a,
491                            nodes_sim=self.nodes_sim,
492                            bias=self.bias,
493                            dropout=self.dropout,
494                            direct_link=self.direct_link,
495                            n_clusters=self.n_clusters,
496                            cluster_encode=self.cluster_encode,
497                            type_clust=self.type_clust,
498                            type_scaling=self.type_scaling,
499                            col_sample=self.col_sample,
500                            row_sample=self.row_sample,
501                            seed=self.seed,
502                            backend=self.backend,
503                        )
504
505                    pipe.fit(X_train, y_train)
506                    self.models[name] = pipe
507                    y_pred = pipe.predict(X_test)
508                    accuracy = accuracy_score(y_test, y_pred, normalize=True)
509                    b_accuracy = balanced_accuracy_score(y_test, y_pred)
510                    f1 = f1_score(y_test, y_pred, average="weighted")
511                    try:
512                        roc_auc = roc_auc_score(y_test, y_pred)
513                    except Exception as exception:
514                        roc_auc = None
515                        if self.ignore_warnings is False:
516                            print("ROC AUC couldn't be calculated for " + name)
517                            print(exception)
518                    names.append(name)
519                    Accuracy.append(accuracy)
520                    B_Accuracy.append(b_accuracy)
521                    ROC_AUC.append(roc_auc)
522                    F1.append(f1)
523                    TIME.append(time.time() - start)
524                    if self.custom_metric is not None:
525                        custom_metric = self.custom_metric(y_test, y_pred)
526                        CUSTOM_METRIC.append(custom_metric)
527                    if self.verbose > 0:
528                        if self.custom_metric is not None:
529                            print(
530                                {
531                                    "Model": name,
532                                    "Accuracy": accuracy,
533                                    "Balanced Accuracy": b_accuracy,
534                                    "ROC AUC": roc_auc,
535                                    "F1 Score": f1,
536                                    self.custom_metric.__name__: custom_metric,
537                                    "Time taken": time.time() - start,
538                                }
539                            )
540                        else:
541                            print(
542                                {
543                                    "Model": name,
544                                    "Accuracy": accuracy,
545                                    "Balanced Accuracy": b_accuracy,
546                                    "ROC AUC": roc_auc,
547                                    "F1 Score": f1,
548                                    "Time taken": time.time() - start,
549                                }
550                            )
551                    if self.predictions:
552                        predictions[name] = y_pred
553                except Exception as exception:
554                    try:
555                        if self.ignore_warnings is False:
556                            print(name + " model failed to execute")
557                            print(exception)
558                    except Exception as exception:
559                        pass
560
561        if self.custom_metric is None:
562            scores = pd.DataFrame(
563                {
564                    "Model": names,
565                    "Accuracy": Accuracy,
566                    "Balanced Accuracy": B_Accuracy,
567                    "ROC AUC": ROC_AUC,
568                    "F1 Score": F1,
569                    "Time Taken": TIME,
570                }
571            )
572        else:
573            scores = pd.DataFrame(
574                {
575                    "Model": names,
576                    "Accuracy": Accuracy,
577                    "Balanced Accuracy": B_Accuracy,
578                    "ROC AUC": ROC_AUC,
579                    "F1 Score": F1,
580                    self.custom_metric.__name__: CUSTOM_METRIC,
581                    "Time Taken": TIME,
582                }
583            )
584        scores = scores.sort_values(by=self.sort_by, ascending=False).set_index(
585            "Model"
586        )
587
588        if self.predictions:
589            predictions_df = pd.DataFrame.from_dict(predictions)
590        return scores, predictions_df if self.predictions is True else scores
591
592    def provide_models(self, X_train, X_test, y_train, y_test):
593        """Returns all the model objects trained. If fit hasn't been called yet,
594        then it's called to return the models.
595
596        Parameters:
597
598        X_train: array-like,
599            Training vectors, where rows is the number of samples
600            and columns is the number of features.
601
602        X_test: array-like,
603            Testing vectors, where rows is the number of samples
604            and columns is the number of features.
605
606        y_train: array-like,
607            Training vectors, where rows is the number of samples
608            and columns is the number of features.
609
610        y_test: array-like,
611            Testing vectors, where rows is the number of samples
612            and columns is the number of features.
613
614        Returns:
615
616            models: dict-object,
617                Returns a dictionary with each model's pipeline as value
618                and key = name of the model.
619        """
620        if len(self.models.keys()) == 0:
621            self.fit(X_train, X_test, y_train, y_test)
622
623        return self.models

Fitting -- almost -- all the classification algorithms with nnetsauce's CustomClassifier and returning their scores.

Parameters:

verbose: int, optional (default=0)
    Any positive number for verbosity.

ignore_warnings: bool, optional (default=True)
    When set to True, warnings related to algorithms that were not
    run are ignored.

custom_metric: function, optional (default=None)
    When function is provided, models are evaluated based on the
    custom evaluation metric provided.

predictions: bool, optional (default=False)
    When set to True, the predictions of all the models models are
    returned as data frame.

sort_by: string, optional (default='Accuracy')
    Sort models by a metric. Available options are 'Accuracy',
    'Balanced Accuracy', 'ROC AUC', 'F1 Score' or a custom metric
    identified by its name and provided by custom_metric.

random_state: int, optional (default=42)
    Reproducibiility seed.

estimators: list, optional (default='all')
    list of Estimators names or just 'all' for > 90
    classifiers (default='all')

preprocess: bool, preprocessing is done when set to True

n_jobs: int, when possible, run in parallel
    For now, only used by individual models that support it.

All the other parameters are the same as CustomClassifier's.

Examples:

import nnetsauce as ns
    from sklearn.datasets import load_breast_cancer
    from sklearn.model_selection import train_test_split
    data = load_breast_cancer()
    X = data.data
    y= data.target
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2,
    random_state=123)
    clf = ns.LazyClassifier(verbose=0, ignore_warnings=True)
    models, predictions = clf.fit(X_train, X_test, y_train, y_test)
    model_dictionary = clf.provide_models(X_train,X_test,y_train,y_test)
    print(models)
    

def fit(self, X_train, X_test, y_train, y_test):
203    def fit(self, X_train, X_test, y_train, y_test):
204        """Fit classifiers to X_train and y_train, predict and score on X_test,
205        y_test.
206
207        Parameters:
208
209            X_train: array-like,
210                Training vectors, where rows is the number of samples
211                and columns is the number of features.
212
213            X_test: array-like,
214                Testing vectors, where rows is the number of samples
215                and columns is the number of features.
216
217            y_train: array-like,
218                Training vectors, where rows is the number of samples
219                and columns is the number of features.
220
221            y_test: array-like,
222                Testing vectors, where rows is the number of samples
223                and columns is the number of features.
224
225        Returns:
226
227            scores: Pandas DataFrame
228                Returns metrics of all the models in a Pandas DataFrame.
229
230            predictions: Pandas DataFrame
231                Returns predictions of all the models in a Pandas DataFrame.
232        """
233        Accuracy = []
234        B_Accuracy = []
235        ROC_AUC = []
236        F1 = []
237        names = []
238        TIME = []
239        predictions = {}
240
241        if self.custom_metric is not None:
242            CUSTOM_METRIC = []
243
244        if isinstance(X_train, np.ndarray):
245            X_train = pd.DataFrame(X_train)
246            X_test = pd.DataFrame(X_test)
247
248        numeric_features = X_train.select_dtypes(include=[np.number]).columns
249        categorical_features = X_train.select_dtypes(include=["object"]).columns
250
251        categorical_low, categorical_high = get_card_split(
252            X_train, categorical_features
253        )
254
255        if self.preprocess is True:
256
257            preprocessor = ColumnTransformer(
258                transformers=[
259                    ("numeric", numeric_transformer, numeric_features),
260                    (
261                        "categorical_low",
262                        categorical_transformer_low,
263                        categorical_low,
264                    ),
265                    (
266                        "categorical_high",
267                        categorical_transformer_high,
268                        categorical_high,
269                    ),
270                ]
271            )
272
273        if self.estimators == "all":
274
275            self.classifiers = (
276                CLASSIFIERS + MULTITASKCLASSIFIERS + SIMPLEMULTITASKCLASSIFIERS
277            )
278
279        else:  # list custom estimators, by their names
280
281            self.classifiers = (
282                [
283                    ("CustomClassifier(" + est[0] + ")", est[1])
284                    for est in all_estimators()
285                    if (
286                        issubclass(est[1], ClassifierMixin)
287                        and (est[0] in self.estimators)
288                    )
289                ]
290                + [
291                    (
292                        "MultitaskClassifier(" + est[0] + ")",
293                        partial(MultitaskClassifier, obj=est[1]()),
294                    )
295                    for est in all_estimators()
296                    if (
297                        issubclass(est[1], RegressorMixin)
298                        and (est[0] in self.estimators)
299                    )
300                ]
301                + [
302                    (
303                        "SimpleMultitaskClassifier(" + est[0] + ")",
304                        partial(SimpleMultitaskClassifier, obj=est[1]()),
305                    )
306                    for est in all_estimators()
307                    if (
308                        issubclass(est[1], RegressorMixin)
309                        and (est[0] in self.estimators)
310                    )
311                ]
312            )
313
314        if self.preprocess is True:
315
316            for name, model in tqdm(self.classifiers):  # do parallel exec
317
318                other_args = (
319                    {}
320                )  # use this trick for `random_state` too --> refactor
321                try:
322                    if (
323                        "n_jobs" in model().get_params().keys()
324                        and name.find("LogisticRegression") == -1
325                    ):
326                        other_args["n_jobs"] = self.n_jobs
327                except Exception:
328                    pass
329
330                start = time.time()
331
332                try:
333                    if "random_state" in model().get_params().keys():
334                        pipe = Pipeline(
335                            [
336                                ("preprocessor", preprocessor),
337                                (
338                                    "classifier",
339                                    CustomClassifier(
340                                        obj=model(
341                                            random_state=self.random_state,
342                                            **other_args
343                                        ),
344                                        n_hidden_features=self.n_hidden_features,
345                                        activation_name=self.activation_name,
346                                        a=self.a,
347                                        nodes_sim=self.nodes_sim,
348                                        bias=self.bias,
349                                        dropout=self.dropout,
350                                        direct_link=self.direct_link,
351                                        n_clusters=self.n_clusters,
352                                        cluster_encode=self.cluster_encode,
353                                        type_clust=self.type_clust,
354                                        type_scaling=self.type_scaling,
355                                        col_sample=self.col_sample,
356                                        row_sample=self.row_sample,
357                                        seed=self.seed,
358                                        backend=self.backend,
359                                    ),
360                                ),
361                            ]
362                        )
363                    else:
364                        pipe = Pipeline(
365                            [
366                                ("preprocessor", preprocessor),
367                                (
368                                    "classifier",
369                                    CustomClassifier(
370                                        obj=model(**other_args),
371                                        n_hidden_features=self.n_hidden_features,
372                                        activation_name=self.activation_name,
373                                        a=self.a,
374                                        nodes_sim=self.nodes_sim,
375                                        bias=self.bias,
376                                        dropout=self.dropout,
377                                        direct_link=self.direct_link,
378                                        n_clusters=self.n_clusters,
379                                        cluster_encode=self.cluster_encode,
380                                        type_clust=self.type_clust,
381                                        type_scaling=self.type_scaling,
382                                        col_sample=self.col_sample,
383                                        row_sample=self.row_sample,
384                                        seed=self.seed,
385                                        backend=self.backend,
386                                    ),
387                                ),
388                            ]
389                        )
390
391                    pipe.fit(X_train, y_train)
392                    self.models[name] = pipe
393                    y_pred = pipe.predict(X_test)
394                    accuracy = accuracy_score(y_test, y_pred, normalize=True)
395                    b_accuracy = balanced_accuracy_score(y_test, y_pred)
396                    f1 = f1_score(y_test, y_pred, average="weighted")
397                    try:
398                        roc_auc = roc_auc_score(y_test, y_pred)
399                    except Exception as exception:
400                        roc_auc = None
401                        if self.ignore_warnings is False:
402                            print("ROC AUC couldn't be calculated for " + name)
403                            print(exception)
404                    names.append(name)
405                    Accuracy.append(accuracy)
406                    B_Accuracy.append(b_accuracy)
407                    ROC_AUC.append(roc_auc)
408                    F1.append(f1)
409                    TIME.append(time.time() - start)
410                    if self.custom_metric is not None:
411                        custom_metric = self.custom_metric(y_test, y_pred)
412                        CUSTOM_METRIC.append(custom_metric)
413                    if self.verbose > 0:
414                        if self.custom_metric is not None:
415                            print(
416                                {
417                                    "Model": name,
418                                    "Accuracy": accuracy,
419                                    "Balanced Accuracy": b_accuracy,
420                                    "ROC AUC": roc_auc,
421                                    "F1 Score": f1,
422                                    self.custom_metric.__name__: custom_metric,
423                                    "Time taken": time.time() - start,
424                                }
425                            )
426                        else:
427                            print(
428                                {
429                                    "Model": name,
430                                    "Accuracy": accuracy,
431                                    "Balanced Accuracy": b_accuracy,
432                                    "ROC AUC": roc_auc,
433                                    "F1 Score": f1,
434                                    "Time taken": time.time() - start,
435                                }
436                            )
437                    if self.predictions:
438                        predictions[name] = y_pred
439                except Exception as exception:
440                    try:
441                        if self.ignore_warnings is False:
442                            print(name + " model failed to execute")
443                            print(exception)
444                    except Exception as exception:
445                        pass
446
447        else:  # if self.preprocess is False:
448
449            for name, model in tqdm(self.classifiers):  # do parallel exec
450                other_args = (
451                    {}
452                )  # use this trick for `random_state` too --> refactor
453                try:
454                    if (
455                        "n_jobs" in model().get_params().keys()
456                        and name.find("LogisticRegression") == -1
457                    ):
458                        other_args["n_jobs"] = self.n_jobs
459                except Exception:
460                    pass
461
462                start = time.time()
463                try:
464                    if "random_state" in model().get_params().keys():
465                        pipe = CustomClassifier(
466                            obj=model(
467                                random_state=self.random_state, **other_args
468                            ),
469                            n_hidden_features=self.n_hidden_features,
470                            activation_name=self.activation_name,
471                            a=self.a,
472                            nodes_sim=self.nodes_sim,
473                            bias=self.bias,
474                            dropout=self.dropout,
475                            direct_link=self.direct_link,
476                            n_clusters=self.n_clusters,
477                            cluster_encode=self.cluster_encode,
478                            type_clust=self.type_clust,
479                            type_scaling=self.type_scaling,
480                            col_sample=self.col_sample,
481                            row_sample=self.row_sample,
482                            seed=self.seed,
483                            backend=self.backend,
484                        )
485                    else:
486                        pipe = CustomClassifier(
487                            obj=model(**other_args),
488                            n_hidden_features=self.n_hidden_features,
489                            activation_name=self.activation_name,
490                            a=self.a,
491                            nodes_sim=self.nodes_sim,
492                            bias=self.bias,
493                            dropout=self.dropout,
494                            direct_link=self.direct_link,
495                            n_clusters=self.n_clusters,
496                            cluster_encode=self.cluster_encode,
497                            type_clust=self.type_clust,
498                            type_scaling=self.type_scaling,
499                            col_sample=self.col_sample,
500                            row_sample=self.row_sample,
501                            seed=self.seed,
502                            backend=self.backend,
503                        )
504
505                    pipe.fit(X_train, y_train)
506                    self.models[name] = pipe
507                    y_pred = pipe.predict(X_test)
508                    accuracy = accuracy_score(y_test, y_pred, normalize=True)
509                    b_accuracy = balanced_accuracy_score(y_test, y_pred)
510                    f1 = f1_score(y_test, y_pred, average="weighted")
511                    try:
512                        roc_auc = roc_auc_score(y_test, y_pred)
513                    except Exception as exception:
514                        roc_auc = None
515                        if self.ignore_warnings is False:
516                            print("ROC AUC couldn't be calculated for " + name)
517                            print(exception)
518                    names.append(name)
519                    Accuracy.append(accuracy)
520                    B_Accuracy.append(b_accuracy)
521                    ROC_AUC.append(roc_auc)
522                    F1.append(f1)
523                    TIME.append(time.time() - start)
524                    if self.custom_metric is not None:
525                        custom_metric = self.custom_metric(y_test, y_pred)
526                        CUSTOM_METRIC.append(custom_metric)
527                    if self.verbose > 0:
528                        if self.custom_metric is not None:
529                            print(
530                                {
531                                    "Model": name,
532                                    "Accuracy": accuracy,
533                                    "Balanced Accuracy": b_accuracy,
534                                    "ROC AUC": roc_auc,
535                                    "F1 Score": f1,
536                                    self.custom_metric.__name__: custom_metric,
537                                    "Time taken": time.time() - start,
538                                }
539                            )
540                        else:
541                            print(
542                                {
543                                    "Model": name,
544                                    "Accuracy": accuracy,
545                                    "Balanced Accuracy": b_accuracy,
546                                    "ROC AUC": roc_auc,
547                                    "F1 Score": f1,
548                                    "Time taken": time.time() - start,
549                                }
550                            )
551                    if self.predictions:
552                        predictions[name] = y_pred
553                except Exception as exception:
554                    try:
555                        if self.ignore_warnings is False:
556                            print(name + " model failed to execute")
557                            print(exception)
558                    except Exception as exception:
559                        pass
560
561        if self.custom_metric is None:
562            scores = pd.DataFrame(
563                {
564                    "Model": names,
565                    "Accuracy": Accuracy,
566                    "Balanced Accuracy": B_Accuracy,
567                    "ROC AUC": ROC_AUC,
568                    "F1 Score": F1,
569                    "Time Taken": TIME,
570                }
571            )
572        else:
573            scores = pd.DataFrame(
574                {
575                    "Model": names,
576                    "Accuracy": Accuracy,
577                    "Balanced Accuracy": B_Accuracy,
578                    "ROC AUC": ROC_AUC,
579                    "F1 Score": F1,
580                    self.custom_metric.__name__: CUSTOM_METRIC,
581                    "Time Taken": TIME,
582                }
583            )
584        scores = scores.sort_values(by=self.sort_by, ascending=False).set_index(
585            "Model"
586        )
587
588        if self.predictions:
589            predictions_df = pd.DataFrame.from_dict(predictions)
590        return scores, predictions_df if self.predictions is True else scores

Fit classifiers to X_train and y_train, predict and score on X_test, y_test.

Parameters:

X_train: array-like,
    Training vectors, where rows is the number of samples
    and columns is the number of features.

X_test: array-like,
    Testing vectors, where rows is the number of samples
    and columns is the number of features.

y_train: array-like,
    Training vectors, where rows is the number of samples
    and columns is the number of features.

y_test: array-like,
    Testing vectors, where rows is the number of samples
    and columns is the number of features.

Returns:

scores: Pandas DataFrame
    Returns metrics of all the models in a Pandas DataFrame.

predictions: Pandas DataFrame
    Returns predictions of all the models in a Pandas DataFrame.
def provide_models(self, X_train, X_test, y_train, y_test):
592    def provide_models(self, X_train, X_test, y_train, y_test):
593        """Returns all the model objects trained. If fit hasn't been called yet,
594        then it's called to return the models.
595
596        Parameters:
597
598        X_train: array-like,
599            Training vectors, where rows is the number of samples
600            and columns is the number of features.
601
602        X_test: array-like,
603            Testing vectors, where rows is the number of samples
604            and columns is the number of features.
605
606        y_train: array-like,
607            Training vectors, where rows is the number of samples
608            and columns is the number of features.
609
610        y_test: array-like,
611            Testing vectors, where rows is the number of samples
612            and columns is the number of features.
613
614        Returns:
615
616            models: dict-object,
617                Returns a dictionary with each model's pipeline as value
618                and key = name of the model.
619        """
620        if len(self.models.keys()) == 0:
621            self.fit(X_train, X_test, y_train, y_test)
622
623        return self.models

Returns all the model objects trained. If fit hasn't been called yet, then it's called to return the models.

Parameters:

X_train: array-like, Training vectors, where rows is the number of samples and columns is the number of features.

X_test: array-like, Testing vectors, where rows is the number of samples and columns is the number of features.

y_train: array-like, Training vectors, where rows is the number of samples and columns is the number of features.

y_test: array-like, Testing vectors, where rows is the number of samples and columns is the number of features.

Returns:

models: dict-object,
    Returns a dictionary with each model's pipeline as value
    and key = name of the model.
class LazyRegressor(nnetsauce.custom.custom.Custom, sklearn.base.RegressorMixin):
 85class LazyRegressor(Custom, RegressorMixin):
 86    """
 87
 88    Fitting a collection of regression models using nnetsauce's CustomRegressor
 89
 90    Parameters:
 91
 92        verbose: int, optional (default=0)
 93            Any positive number for verbosity.
 94
 95        ignore_warnings: bool, optional (default=True)
 96            When set to True, the warning related to algorigms that are not able to run are ignored.
 97
 98        custom_metric: function, optional (default=None)
 99            When function is provided, models are evaluated based on the custom evaluation metric provided.
100
101        predictions: bool, optional (default=False)
102            When set to True, the predictions of all the models models are returned as dataframe.
103
104        sort_by: string, optional (default='Accuracy')
105            Sort models by a metric. Available options are 'Accuracy', 'Balanced Accuracy', 'ROC AUC', 'F1 Score'
106            or a custom metric identified by its name and provided by custom_metric.
107
108        random_state: int, optional (default=42)
109            Reproducibiility seed.
110
111        estimators: list, optional (default='all')
112            a list of Estimators names or just 'all' (default='all')
113
114        preprocess: bool
115            preprocessing is done when set to True
116
117        n_jobs : int, when possible, run in parallel
118            For now, only used by individual models that support it.
119
120        n_layers: int, optional (default=3)
121            Number of layers of CustomRegressors to be used.
122
123        All the other parameters are the same as CustomRegressor's.
124
125    Examples:
126        ```python
127        import nnetsauce as ns
128        from sklearn.datasets import load_diabetes
129        from sklearn.model_selection import train_test_split
130        data = load_diabetes()
131        X = data.data
132        y= data.target
133        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123)
134        regr = ns.LazyRegressor(verbose=0, ignore_warnings=True)
135        models, predictions = clf.fit(X_train, X_test, y_train, y_test)
136        model_dictionary = clf.provide_models(X_train,X_test,y_train,y_test)
137        print(models)
138        ```
139
140    """
141
142    def __init__(
143        self,
144        verbose=0,
145        ignore_warnings=True,
146        custom_metric=None,
147        predictions=False,
148        random_state=42,
149        estimators="all",
150        preprocess=False,
151        n_jobs=None,
152        # CustomRegressor attributes
153        obj=None,
154        n_hidden_features=5,
155        activation_name="relu",
156        a=0.01,
157        nodes_sim="sobol",
158        bias=True,
159        dropout=0,
160        direct_link=True,
161        n_clusters=2,
162        cluster_encode=True,
163        type_clust="kmeans",
164        type_scaling=("std", "std", "std"),
165        col_sample=1,
166        row_sample=1,
167        seed=123,
168        backend="cpu",
169    ):
170        self.verbose = verbose
171        self.ignore_warnings = ignore_warnings
172        self.custom_metric = custom_metric
173        self.predictions = predictions
174        self.models = {}
175        self.random_state = random_state
176        self.estimators = estimators
177        self.preprocess = preprocess
178        self.n_jobs = n_jobs
179        super().__init__(
180            obj=obj,
181            n_hidden_features=n_hidden_features,
182            activation_name=activation_name,
183            a=a,
184            nodes_sim=nodes_sim,
185            bias=bias,
186            dropout=dropout,
187            direct_link=direct_link,
188            n_clusters=n_clusters,
189            cluster_encode=cluster_encode,
190            type_clust=type_clust,
191            type_scaling=type_scaling,
192            col_sample=col_sample,
193            row_sample=row_sample,
194            seed=seed,
195            backend=backend,
196        )
197
198    def fit(self, X_train, X_test, y_train, y_test):
199        """Fit Regression algorithms to X_train and y_train, predict and score on X_test, y_test.
200
201        Parameters:
202
203            X_train : array-like,
204                Training vectors, where rows is the number of samples
205                and columns is the number of features.
206
207            X_test : array-like,
208                Testing vectors, where rows is the number of samples
209                and columns is the number of features.
210
211            y_train : array-like,
212                Training vectors, where rows is the number of samples
213                and columns is the number of features.
214
215            y_test : array-like,
216                Testing vectors, where rows is the number of samples
217                and columns is the number of features.
218
219        Returns:
220
221        scores : Pandas DataFrame
222            Returns metrics of all the models in a Pandas DataFrame.
223
224        predictions : Pandas DataFrame
225            Returns predictions of all the models in a Pandas DataFrame.
226
227        """
228        R2 = []
229        ADJR2 = []
230        RMSE = []
231        # WIN = []
232        names = []
233        TIME = []
234        predictions = {}
235
236        if self.custom_metric:
237            CUSTOM_METRIC = []
238
239        if isinstance(X_train, np.ndarray):
240            X_train = pd.DataFrame(X_train)
241            X_test = pd.DataFrame(X_test)
242
243        numeric_features = X_train.select_dtypes(include=[np.number]).columns
244        categorical_features = X_train.select_dtypes(include=["object"]).columns
245
246        categorical_low, categorical_high = get_card_split(
247            X_train, categorical_features
248        )
249
250        if self.preprocess is True:
251            preprocessor = ColumnTransformer(
252                transformers=[
253                    ("numeric", numeric_transformer, numeric_features),
254                    (
255                        "categorical_low",
256                        categorical_transformer_low,
257                        categorical_low,
258                    ),
259                    (
260                        "categorical_high",
261                        categorical_transformer_high,
262                        categorical_high,
263                    ),
264                ]
265            )
266
267        if self.estimators == "all":
268            self.regressors = REGRESSORS
269        else:
270            self.regressors = [
271                ("CustomRegressor(" + est[0] + ")", est[1])
272                for est in all_estimators()
273                if (
274                    issubclass(est[1], RegressorMixin)
275                    and (est[0] in self.estimators)
276                )
277            ]
278
279        if self.preprocess is True:
280            for name, model in tqdm(self.regressors):  # do parallel exec
281                start = time.time()
282                try:
283                    if "random_state" in model().get_params().keys():
284                        pipe = Pipeline(
285                            steps=[
286                                ("preprocessor", preprocessor),
287                                (
288                                    "regressor",
289                                    CustomRegressor(
290                                        obj=model(
291                                            random_state=self.random_state
292                                        ),
293                                        n_hidden_features=self.n_hidden_features,
294                                        activation_name=self.activation_name,
295                                        a=self.a,
296                                        nodes_sim=self.nodes_sim,
297                                        bias=self.bias,
298                                        dropout=self.dropout,
299                                        direct_link=self.direct_link,
300                                        n_clusters=self.n_clusters,
301                                        cluster_encode=self.cluster_encode,
302                                        type_clust=self.type_clust,
303                                        type_scaling=self.type_scaling,
304                                        col_sample=self.col_sample,
305                                        row_sample=self.row_sample,
306                                        seed=self.seed,
307                                        backend=self.backend,
308                                    ),
309                                ),
310                            ]
311                        )
312                    else:
313                        pipe = Pipeline(
314                            steps=[
315                                ("preprocessor", preprocessor),
316                                (
317                                    "regressor",
318                                    CustomRegressor(
319                                        obj=model(),
320                                        n_hidden_features=self.n_hidden_features,
321                                        activation_name=self.activation_name,
322                                        a=self.a,
323                                        nodes_sim=self.nodes_sim,
324                                        bias=self.bias,
325                                        dropout=self.dropout,
326                                        direct_link=self.direct_link,
327                                        n_clusters=self.n_clusters,
328                                        cluster_encode=self.cluster_encode,
329                                        type_clust=self.type_clust,
330                                        type_scaling=self.type_scaling,
331                                        col_sample=self.col_sample,
332                                        row_sample=self.row_sample,
333                                        seed=self.seed,
334                                        backend=self.backend,
335                                    ),
336                                ),
337                            ]
338                        )
339
340                    pipe.fit(X_train, y_train)
341                    self.models[name] = pipe
342                    y_pred = pipe.predict(X_test)
343
344                    r_squared = r2_score(y_test, y_pred)
345                    adj_rsquared = adjusted_rsquared(
346                        r_squared, X_test.shape[0], X_test.shape[1]
347                    )
348                    rmse = mean_squared_error(y_test, y_pred, squared=False)
349
350                    names.append(name)
351                    R2.append(r_squared)
352                    ADJR2.append(adj_rsquared)
353                    RMSE.append(rmse)
354                    TIME.append(time.time() - start)
355
356                    if self.custom_metric:
357                        custom_metric = self.custom_metric(y_test, y_pred)
358                        CUSTOM_METRIC.append(custom_metric)
359
360                    if self.verbose > 0:
361                        scores_verbose = {
362                            "Model": name,
363                            "R-Squared": r_squared,
364                            "Adjusted R-Squared": adj_rsquared,
365                            "RMSE": rmse,
366                            "Time taken": time.time() - start,
367                        }
368
369                        if self.custom_metric:
370                            scores_verbose[self.custom_metric.__name__] = (
371                                custom_metric
372                            )
373
374                        print(scores_verbose)
375                    if self.predictions:
376                        predictions[name] = y_pred
377                except Exception as exception:
378                    if self.ignore_warnings is False:
379                        print(name + " model failed to execute")
380                        print(exception)
381
382        else:
383            for name, model in tqdm(self.regressors):  # do parallel exec
384                start = time.time()
385                try:
386                    if "random_state" in model().get_params().keys():
387                        pipe = CustomRegressor(
388                            obj=model(random_state=self.random_state),
389                            n_hidden_features=self.n_hidden_features,
390                            activation_name=self.activation_name,
391                            a=self.a,
392                            nodes_sim=self.nodes_sim,
393                            bias=self.bias,
394                            dropout=self.dropout,
395                            direct_link=self.direct_link,
396                            n_clusters=self.n_clusters,
397                            cluster_encode=self.cluster_encode,
398                            type_clust=self.type_clust,
399                            type_scaling=self.type_scaling,
400                            col_sample=self.col_sample,
401                            row_sample=self.row_sample,
402                            seed=self.seed,
403                            backend=self.backend,
404                        )
405                    else:
406                        pipe = CustomRegressor(
407                            obj=model(),
408                            n_hidden_features=self.n_hidden_features,
409                            activation_name=self.activation_name,
410                            a=self.a,
411                            nodes_sim=self.nodes_sim,
412                            bias=self.bias,
413                            dropout=self.dropout,
414                            direct_link=self.direct_link,
415                            n_clusters=self.n_clusters,
416                            cluster_encode=self.cluster_encode,
417                            type_clust=self.type_clust,
418                            type_scaling=self.type_scaling,
419                            col_sample=self.col_sample,
420                            row_sample=self.row_sample,
421                            seed=self.seed,
422                            backend=self.backend,
423                        )
424
425                    pipe.fit(X_train, y_train)
426                    self.models[name] = pipe
427                    y_pred = pipe.predict(X_test)
428
429                    r_squared = r2_score(y_test, y_pred)
430                    adj_rsquared = adjusted_rsquared(
431                        r_squared, X_test.shape[0], X_test.shape[1]
432                    )
433                    rmse = mean_squared_error(y_test, y_pred, squared=False)
434
435                    names.append(name)
436                    R2.append(r_squared)
437                    ADJR2.append(adj_rsquared)
438                    RMSE.append(rmse)
439                    TIME.append(time.time() - start)
440
441                    if self.custom_metric:
442                        custom_metric = self.custom_metric(y_test, y_pred)
443                        CUSTOM_METRIC.append(custom_metric)
444
445                    if self.verbose > 0:
446                        scores_verbose = {
447                            "Model": name,
448                            "R-Squared": r_squared,
449                            "Adjusted R-Squared": adj_rsquared,
450                            "RMSE": rmse,
451                            "Time taken": time.time() - start,
452                        }
453
454                        if self.custom_metric:
455                            scores_verbose[self.custom_metric.__name__] = (
456                                custom_metric
457                            )
458
459                        print(scores_verbose)
460                    if self.predictions:
461                        predictions[name] = y_pred
462                except Exception as exception:
463                    if self.ignore_warnings is False:
464                        print(name + " model failed to execute")
465                        print(exception)
466
467        scores = {
468            "Model": names,
469            "Adjusted R-Squared": ADJR2,
470            "R-Squared": R2,
471            "RMSE": RMSE,
472            "Time Taken": TIME,
473        }
474
475        if self.custom_metric:
476            scores[self.custom_metric.__name__] = CUSTOM_METRIC
477
478        scores = pd.DataFrame(scores)
479        scores = scores.sort_values(by="RMSE", ascending=True).set_index(
480            "Model"
481        )
482
483        if self.predictions:
484            predictions_df = pd.DataFrame.from_dict(predictions)
485        return scores, predictions_df if self.predictions is True else scores
486
487    def provide_models(self, X_train, X_test, y_train, y_test):
488        """
489        This function returns all the model objects trained in fit function.
490        If fit is not called already, then we call fit and then return the models.
491        Parameters
492        ----------
493        X_train : array-like,
494            Training vectors, where rows is the number of samples
495            and columns is the number of features.
496        X_test : array-like,
497            Testing vectors, where rows is the number of samples
498            and columns is the number of features.
499        y_train : array-like,
500            Training vectors, where rows is the number of samples
501            and columns is the number of features.
502        y_test : array-like,
503            Testing vectors, where rows is the number of samples
504            and columns is the number of features.
505        Returns
506        -------
507        models: dict-object,
508            Returns a dictionary with each model pipeline as value
509            with key as name of models.
510        """
511        if len(self.models.keys()) == 0:
512            self.fit(X_train, X_test, y_train, y_test)
513
514        return self.models

Fitting a collection of regression models using nnetsauce's CustomRegressor

Parameters:

verbose: int, optional (default=0)
    Any positive number for verbosity.

ignore_warnings: bool, optional (default=True)
    When set to True, the warning related to algorigms that are not able to run are ignored.

custom_metric: function, optional (default=None)
    When function is provided, models are evaluated based on the custom evaluation metric provided.

predictions: bool, optional (default=False)
    When set to True, the predictions of all the models models are returned as dataframe.

sort_by: string, optional (default='Accuracy')
    Sort models by a metric. Available options are 'Accuracy', 'Balanced Accuracy', 'ROC AUC', 'F1 Score'
    or a custom metric identified by its name and provided by custom_metric.

random_state: int, optional (default=42)
    Reproducibiility seed.

estimators: list, optional (default='all')
    a list of Estimators names or just 'all' (default='all')

preprocess: bool
    preprocessing is done when set to True

n_jobs : int, when possible, run in parallel
    For now, only used by individual models that support it.

n_layers: int, optional (default=3)
    Number of layers of CustomRegressors to be used.

All the other parameters are the same as CustomRegressor's.

Examples:

import nnetsauce as ns
    from sklearn.datasets import load_diabetes
    from sklearn.model_selection import train_test_split
    data = load_diabetes()
    X = data.data
    y= data.target
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123)
    regr = ns.LazyRegressor(verbose=0, ignore_warnings=True)
    models, predictions = clf.fit(X_train, X_test, y_train, y_test)
    model_dictionary = clf.provide_models(X_train,X_test,y_train,y_test)
    print(models)
    

def fit(self, X_train, X_test, y_train, y_test):
198    def fit(self, X_train, X_test, y_train, y_test):
199        """Fit Regression algorithms to X_train and y_train, predict and score on X_test, y_test.
200
201        Parameters:
202
203            X_train : array-like,
204                Training vectors, where rows is the number of samples
205                and columns is the number of features.
206
207            X_test : array-like,
208                Testing vectors, where rows is the number of samples
209                and columns is the number of features.
210
211            y_train : array-like,
212                Training vectors, where rows is the number of samples
213                and columns is the number of features.
214
215            y_test : array-like,
216                Testing vectors, where rows is the number of samples
217                and columns is the number of features.
218
219        Returns:
220
221        scores : Pandas DataFrame
222            Returns metrics of all the models in a Pandas DataFrame.
223
224        predictions : Pandas DataFrame
225            Returns predictions of all the models in a Pandas DataFrame.
226
227        """
228        R2 = []
229        ADJR2 = []
230        RMSE = []
231        # WIN = []
232        names = []
233        TIME = []
234        predictions = {}
235
236        if self.custom_metric:
237            CUSTOM_METRIC = []
238
239        if isinstance(X_train, np.ndarray):
240            X_train = pd.DataFrame(X_train)
241            X_test = pd.DataFrame(X_test)
242
243        numeric_features = X_train.select_dtypes(include=[np.number]).columns
244        categorical_features = X_train.select_dtypes(include=["object"]).columns
245
246        categorical_low, categorical_high = get_card_split(
247            X_train, categorical_features
248        )
249
250        if self.preprocess is True:
251            preprocessor = ColumnTransformer(
252                transformers=[
253                    ("numeric", numeric_transformer, numeric_features),
254                    (
255                        "categorical_low",
256                        categorical_transformer_low,
257                        categorical_low,
258                    ),
259                    (
260                        "categorical_high",
261                        categorical_transformer_high,
262                        categorical_high,
263                    ),
264                ]
265            )
266
267        if self.estimators == "all":
268            self.regressors = REGRESSORS
269        else:
270            self.regressors = [
271                ("CustomRegressor(" + est[0] + ")", est[1])
272                for est in all_estimators()
273                if (
274                    issubclass(est[1], RegressorMixin)
275                    and (est[0] in self.estimators)
276                )
277            ]
278
279        if self.preprocess is True:
280            for name, model in tqdm(self.regressors):  # do parallel exec
281                start = time.time()
282                try:
283                    if "random_state" in model().get_params().keys():
284                        pipe = Pipeline(
285                            steps=[
286                                ("preprocessor", preprocessor),
287                                (
288                                    "regressor",
289                                    CustomRegressor(
290                                        obj=model(
291                                            random_state=self.random_state
292                                        ),
293                                        n_hidden_features=self.n_hidden_features,
294                                        activation_name=self.activation_name,
295                                        a=self.a,
296                                        nodes_sim=self.nodes_sim,
297                                        bias=self.bias,
298                                        dropout=self.dropout,
299                                        direct_link=self.direct_link,
300                                        n_clusters=self.n_clusters,
301                                        cluster_encode=self.cluster_encode,
302                                        type_clust=self.type_clust,
303                                        type_scaling=self.type_scaling,
304                                        col_sample=self.col_sample,
305                                        row_sample=self.row_sample,
306                                        seed=self.seed,
307                                        backend=self.backend,
308                                    ),
309                                ),
310                            ]
311                        )
312                    else:
313                        pipe = Pipeline(
314                            steps=[
315                                ("preprocessor", preprocessor),
316                                (
317                                    "regressor",
318                                    CustomRegressor(
319                                        obj=model(),
320                                        n_hidden_features=self.n_hidden_features,
321                                        activation_name=self.activation_name,
322                                        a=self.a,
323                                        nodes_sim=self.nodes_sim,
324                                        bias=self.bias,
325                                        dropout=self.dropout,
326                                        direct_link=self.direct_link,
327                                        n_clusters=self.n_clusters,
328                                        cluster_encode=self.cluster_encode,
329                                        type_clust=self.type_clust,
330                                        type_scaling=self.type_scaling,
331                                        col_sample=self.col_sample,
332                                        row_sample=self.row_sample,
333                                        seed=self.seed,
334                                        backend=self.backend,
335                                    ),
336                                ),
337                            ]
338                        )
339
340                    pipe.fit(X_train, y_train)
341                    self.models[name] = pipe
342                    y_pred = pipe.predict(X_test)
343
344                    r_squared = r2_score(y_test, y_pred)
345                    adj_rsquared = adjusted_rsquared(
346                        r_squared, X_test.shape[0], X_test.shape[1]
347                    )
348                    rmse = mean_squared_error(y_test, y_pred, squared=False)
349
350                    names.append(name)
351                    R2.append(r_squared)
352                    ADJR2.append(adj_rsquared)
353                    RMSE.append(rmse)
354                    TIME.append(time.time() - start)
355
356                    if self.custom_metric:
357                        custom_metric = self.custom_metric(y_test, y_pred)
358                        CUSTOM_METRIC.append(custom_metric)
359
360                    if self.verbose > 0:
361                        scores_verbose = {
362                            "Model": name,
363                            "R-Squared": r_squared,
364                            "Adjusted R-Squared": adj_rsquared,
365                            "RMSE": rmse,
366                            "Time taken": time.time() - start,
367                        }
368
369                        if self.custom_metric:
370                            scores_verbose[self.custom_metric.__name__] = (
371                                custom_metric
372                            )
373
374                        print(scores_verbose)
375                    if self.predictions:
376                        predictions[name] = y_pred
377                except Exception as exception:
378                    if self.ignore_warnings is False:
379                        print(name + " model failed to execute")
380                        print(exception)
381
382        else:
383            for name, model in tqdm(self.regressors):  # do parallel exec
384                start = time.time()
385                try:
386                    if "random_state" in model().get_params().keys():
387                        pipe = CustomRegressor(
388                            obj=model(random_state=self.random_state),
389                            n_hidden_features=self.n_hidden_features,
390                            activation_name=self.activation_name,
391                            a=self.a,
392                            nodes_sim=self.nodes_sim,
393                            bias=self.bias,
394                            dropout=self.dropout,
395                            direct_link=self.direct_link,
396                            n_clusters=self.n_clusters,
397                            cluster_encode=self.cluster_encode,
398                            type_clust=self.type_clust,
399                            type_scaling=self.type_scaling,
400                            col_sample=self.col_sample,
401                            row_sample=self.row_sample,
402                            seed=self.seed,
403                            backend=self.backend,
404                        )
405                    else:
406                        pipe = CustomRegressor(
407                            obj=model(),
408                            n_hidden_features=self.n_hidden_features,
409                            activation_name=self.activation_name,
410                            a=self.a,
411                            nodes_sim=self.nodes_sim,
412                            bias=self.bias,
413                            dropout=self.dropout,
414                            direct_link=self.direct_link,
415                            n_clusters=self.n_clusters,
416                            cluster_encode=self.cluster_encode,
417                            type_clust=self.type_clust,
418                            type_scaling=self.type_scaling,
419                            col_sample=self.col_sample,
420                            row_sample=self.row_sample,
421                            seed=self.seed,
422                            backend=self.backend,
423                        )
424
425                    pipe.fit(X_train, y_train)
426                    self.models[name] = pipe
427                    y_pred = pipe.predict(X_test)
428
429                    r_squared = r2_score(y_test, y_pred)
430                    adj_rsquared = adjusted_rsquared(
431                        r_squared, X_test.shape[0], X_test.shape[1]
432                    )
433                    rmse = mean_squared_error(y_test, y_pred, squared=False)
434
435                    names.append(name)
436                    R2.append(r_squared)
437                    ADJR2.append(adj_rsquared)
438                    RMSE.append(rmse)
439                    TIME.append(time.time() - start)
440
441                    if self.custom_metric:
442                        custom_metric = self.custom_metric(y_test, y_pred)
443                        CUSTOM_METRIC.append(custom_metric)
444
445                    if self.verbose > 0:
446                        scores_verbose = {
447                            "Model": name,
448                            "R-Squared": r_squared,
449                            "Adjusted R-Squared": adj_rsquared,
450                            "RMSE": rmse,
451                            "Time taken": time.time() - start,
452                        }
453
454                        if self.custom_metric:
455                            scores_verbose[self.custom_metric.__name__] = (
456                                custom_metric
457                            )
458
459                        print(scores_verbose)
460                    if self.predictions:
461                        predictions[name] = y_pred
462                except Exception as exception:
463                    if self.ignore_warnings is False:
464                        print(name + " model failed to execute")
465                        print(exception)
466
467        scores = {
468            "Model": names,
469            "Adjusted R-Squared": ADJR2,
470            "R-Squared": R2,
471            "RMSE": RMSE,
472            "Time Taken": TIME,
473        }
474
475        if self.custom_metric:
476            scores[self.custom_metric.__name__] = CUSTOM_METRIC
477
478        scores = pd.DataFrame(scores)
479        scores = scores.sort_values(by="RMSE", ascending=True).set_index(
480            "Model"
481        )
482
483        if self.predictions:
484            predictions_df = pd.DataFrame.from_dict(predictions)
485        return scores, predictions_df if self.predictions is True else scores

Fit Regression algorithms to X_train and y_train, predict and score on X_test, y_test.

Parameters:

X_train : array-like,
    Training vectors, where rows is the number of samples
    and columns is the number of features.

X_test : array-like,
    Testing vectors, where rows is the number of samples
    and columns is the number of features.

y_train : array-like,
    Training vectors, where rows is the number of samples
    and columns is the number of features.

y_test : array-like,
    Testing vectors, where rows is the number of samples
    and columns is the number of features.

Returns:

scores : Pandas DataFrame Returns metrics of all the models in a Pandas DataFrame.

predictions : Pandas DataFrame Returns predictions of all the models in a Pandas DataFrame.

def provide_models(self, X_train, X_test, y_train, y_test):
487    def provide_models(self, X_train, X_test, y_train, y_test):
488        """
489        This function returns all the model objects trained in fit function.
490        If fit is not called already, then we call fit and then return the models.
491        Parameters
492        ----------
493        X_train : array-like,
494            Training vectors, where rows is the number of samples
495            and columns is the number of features.
496        X_test : array-like,
497            Testing vectors, where rows is the number of samples
498            and columns is the number of features.
499        y_train : array-like,
500            Training vectors, where rows is the number of samples
501            and columns is the number of features.
502        y_test : array-like,
503            Testing vectors, where rows is the number of samples
504            and columns is the number of features.
505        Returns
506        -------
507        models: dict-object,
508            Returns a dictionary with each model pipeline as value
509            with key as name of models.
510        """
511        if len(self.models.keys()) == 0:
512            self.fit(X_train, X_test, y_train, y_test)
513
514        return self.models

This function returns all the model objects trained in fit function. If fit is not called already, then we call fit and then return the models.

Parameters

X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features. y_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.

Returns

models: dict-object, Returns a dictionary with each model pipeline as value with key as name of models.

class LazyDeepClassifier(nnetsauce.custom.custom.Custom, sklearn.base.ClassifierMixin):
 84class LazyDeepClassifier(Custom, ClassifierMixin):
 85    """
 86
 87    Fitting -- almost -- all the classification algorithms with layers of
 88    nnetsauce's CustomClassifier and returning their scores.
 89
 90    Parameters:
 91
 92        verbose: int, optional (default=0)
 93            Any positive number for verbosity.
 94
 95        ignore_warnings: bool, optional (default=True)
 96            When set to True, the warning related to algorigms that are not
 97            able to run are ignored.
 98
 99        custom_metric: function, optional (default=None)
100            When function is provided, models are evaluated based on the custom
101              evaluation metric provided.
102
103        predictions: bool, optional (default=False)
104            When set to True, the predictions of all the models models are
105            returned as data frame.
106
107        sort_by: string, optional (default='Accuracy')
108            Sort models by a metric. Available options are 'Accuracy',
109            'Balanced Accuracy', 'ROC AUC', 'F1 Score' or a custom metric
110            identified by its name and provided by custom_metric.
111
112        random_state: int, optional (default=42)
113            Reproducibiility seed.
114
115        estimators: list, optional (default='all')
116            list of Estimators names or just 'all' for > 90 classifiers
117            (default='all')
118
119        preprocess: bool, preprocessing is done when set to True
120
121        n_jobs: int, when possible, run in parallel
122            For now, only used by individual models that support it.
123
124        n_layers: int, optional (default=3)
125            Number of layers of CustomClassifiers to be used.
126
127        All the other parameters are the same as CustomClassifier's.
128
129
130    Examples
131
132        ```python
133        import nnetsauce as ns
134        from sklearn.datasets import load_breast_cancer
135        from sklearn.model_selection import train_test_split
136        data = load_breast_cancer()
137        X = data.data
138        y= data.target
139        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2,
140            random_state=123)
141        clf = ns.LazyDeepClassifier(verbose=0, ignore_warnings=True, custom_metric=None)
142        models, predictions = clf.fit(X_train, X_test, y_train, y_test)
143        model_dictionary = clf.provide_models(X_train,X_test,y_train,y_test)
144        print(models)
145        ```
146
147    """
148
149    def __init__(
150        self,
151        verbose=0,
152        ignore_warnings=True,
153        custom_metric=None,
154        predictions=False,
155        sort_by="Accuracy",
156        random_state=42,
157        estimators="all",
158        preprocess=False,
159        n_jobs=None,
160        # Defining depth
161        n_layers=3,
162        # CustomClassifier attributes
163        obj=None,
164        n_hidden_features=5,
165        activation_name="relu",
166        a=0.01,
167        nodes_sim="sobol",
168        bias=True,
169        dropout=0,
170        direct_link=True,
171        n_clusters=2,
172        cluster_encode=True,
173        type_clust="kmeans",
174        type_scaling=("std", "std", "std"),
175        col_sample=1,
176        row_sample=1,
177        seed=123,
178        backend="cpu",
179    ):
180        self.verbose = verbose
181        self.ignore_warnings = ignore_warnings
182        self.custom_metric = custom_metric
183        self.predictions = predictions
184        self.sort_by = sort_by
185        self.models = {}
186        self.random_state = random_state
187        self.estimators = estimators
188        self.preprocess = preprocess
189        self.n_layers = n_layers - 1
190        self.n_jobs = n_jobs
191        super().__init__(
192            obj=obj,
193            n_hidden_features=n_hidden_features,
194            activation_name=activation_name,
195            a=a,
196            nodes_sim=nodes_sim,
197            bias=bias,
198            dropout=dropout,
199            direct_link=direct_link,
200            n_clusters=n_clusters,
201            cluster_encode=cluster_encode,
202            type_clust=type_clust,
203            type_scaling=type_scaling,
204            col_sample=col_sample,
205            row_sample=row_sample,
206            seed=seed,
207            backend=backend,
208        )
209
210    def fit(self, X_train, X_test, y_train, y_test):
211        """Fit classifiers to X_train and y_train, predict and score on X_test,
212        y_test.
213
214        Parameters:
215
216            X_train: array-like,
217                Training vectors, where rows is the number of samples
218                and columns is the number of features.
219
220            X_test: array-like,
221                Testing vectors, where rows is the number of samples
222                and columns is the number of features.
223
224            y_train: array-like,
225                Training vectors, where rows is the number of samples
226                and columns is the number of features.
227
228            y_test: array-like,
229                Testing vectors, where rows is the number of samples
230                and columns is the number of features.
231
232        Returns:
233
234            scores: Pandas DataFrame
235                Returns metrics of all the models in a Pandas DataFrame.
236
237            predictions: Pandas DataFrame
238                Returns predictions of all the models in a Pandas DataFrame.
239        """
240        Accuracy = []
241        B_Accuracy = []
242        ROC_AUC = []
243        F1 = []
244        names = []
245        TIME = []
246        predictions = {}
247
248        if self.custom_metric is not None:
249            CUSTOM_METRIC = []
250
251        if isinstance(X_train, np.ndarray):
252            X_train = pd.DataFrame(X_train)
253            X_test = pd.DataFrame(X_test)
254
255        numeric_features = X_train.select_dtypes(include=[np.number]).columns
256        categorical_features = X_train.select_dtypes(include=["object"]).columns
257
258        categorical_low, categorical_high = get_card_split(
259            X_train, categorical_features
260        )
261
262        if self.preprocess is True:
263            preprocessor = ColumnTransformer(
264                transformers=[
265                    ("numeric", numeric_transformer, numeric_features),
266                    (
267                        "categorical_low",
268                        categorical_transformer_low,
269                        categorical_low,
270                    ),
271                    (
272                        "categorical_high",
273                        categorical_transformer_high,
274                        categorical_high,
275                    ),
276                ]
277            )
278
279        if self.estimators == "all":
280            self.classifiers = [
281                item
282                for sublist in [
283                    DEEPCLASSIFIERS,
284                    DEEPMULTITASKCLASSIFIERS,
285                    DEEPSIMPLEMULTITASKCLASSIFIERS,
286                ]
287                for item in sublist
288            ]
289        else:
290            self.classifiers = (
291                [
292                    ("DeepCustomClassifier(" + est[0] + ")", est[1])
293                    for est in all_estimators()
294                    if (
295                        issubclass(est[1], ClassifierMixin)
296                        and (est[0] in self.estimators)
297                    )
298                ]
299                + [
300                    (
301                        "DeepMultitaskClassifier(" + est[0] + ")",
302                        partial(MultitaskClassifier, obj=est[1]()),
303                    )
304                    for est in all_estimators()
305                    if (
306                        issubclass(est[1], RegressorMixin)
307                        and (est[0] in self.estimators)
308                    )
309                ]
310                + [
311                    (
312                        "DeepSimpleMultitaskClassifier(" + est[0] + ")",
313                        partial(SimpleMultitaskClassifier, obj=est[1]()),
314                    )
315                    for est in all_estimators()
316                    if (
317                        issubclass(est[1], RegressorMixin)
318                        and (est[0] in self.estimators)
319                    )
320                ]
321            )
322
323        if self.preprocess is True:
324
325            for name, model in tqdm(self.classifiers):  # do parallel exec
326
327                other_args = (
328                    {}
329                )  # use this trick for `random_state` too --> refactor
330                try:
331                    if (
332                        "n_jobs" in model().get_params().keys()
333                        and name.find("LogisticRegression") == -1
334                    ):
335                        other_args["n_jobs"] = self.n_jobs
336                except Exception:
337                    pass
338
339                start = time.time()
340
341                try:
342                    if "random_state" in model().get_params().keys():
343                        layer_clf = CustomClassifier(
344                            obj=model(random_state=self.random_state),
345                            n_hidden_features=self.n_hidden_features,
346                            activation_name=self.activation_name,
347                            a=self.a,
348                            nodes_sim=self.nodes_sim,
349                            bias=self.bias,
350                            dropout=self.dropout,
351                            direct_link=self.direct_link,
352                            n_clusters=self.n_clusters,
353                            cluster_encode=self.cluster_encode,
354                            type_clust=self.type_clust,
355                            type_scaling=self.type_scaling,
356                            col_sample=self.col_sample,
357                            row_sample=self.row_sample,
358                            seed=self.seed,
359                            backend=self.backend,
360                        )
361
362                    else:
363                        layer_clf = CustomClassifier(
364                            obj=model(),
365                            n_hidden_features=self.n_hidden_features,
366                            activation_name=self.activation_name,
367                            a=self.a,
368                            nodes_sim=self.nodes_sim,
369                            bias=self.bias,
370                            dropout=self.dropout,
371                            direct_link=self.direct_link,
372                            n_clusters=self.n_clusters,
373                            cluster_encode=self.cluster_encode,
374                            type_clust=self.type_clust,
375                            type_scaling=self.type_scaling,
376                            col_sample=self.col_sample,
377                            row_sample=self.row_sample,
378                            seed=self.seed,
379                            backend=self.backend,
380                        )
381
382                    layer_clf.fit(X_train, y_train)
383
384                    for _ in range(self.n_layers):
385                        layer_clf = deepcopy(
386                            CustomClassifier(
387                                obj=layer_clf,
388                                n_hidden_features=self.n_hidden_features,
389                                activation_name=self.activation_name,
390                                a=self.a,
391                                nodes_sim=self.nodes_sim,
392                                bias=self.bias,
393                                dropout=self.dropout,
394                                direct_link=self.direct_link,
395                                n_clusters=self.n_clusters,
396                                cluster_encode=self.cluster_encode,
397                                type_clust=self.type_clust,
398                                type_scaling=self.type_scaling,
399                                col_sample=self.col_sample,
400                                row_sample=self.row_sample,
401                                seed=self.seed,
402                                backend=self.backend,
403                            )
404                        )
405
406                    pipe = Pipeline(
407                        [
408                            ("preprocessor", preprocessor),
409                            ("classifier", layer_clf),
410                        ]
411                    )
412
413                    pipe.fit(X_train, y_train)
414                    self.models[name] = pipe
415                    y_pred = pipe.predict(X_test)
416                    accuracy = accuracy_score(y_test, y_pred, normalize=True)
417                    b_accuracy = balanced_accuracy_score(y_test, y_pred)
418                    f1 = f1_score(y_test, y_pred, average="weighted")
419                    try:
420                        roc_auc = roc_auc_score(y_test, y_pred)
421                    except Exception as exception:
422                        roc_auc = None
423                        if self.ignore_warnings is False:
424                            print("ROC AUC couldn't be calculated for " + name)
425                            print(exception)
426                    names.append(name)
427                    Accuracy.append(accuracy)
428                    B_Accuracy.append(b_accuracy)
429                    ROC_AUC.append(roc_auc)
430                    F1.append(f1)
431                    TIME.append(time.time() - start)
432                    if self.custom_metric is not None:
433                        custom_metric = self.custom_metric(y_test, y_pred)
434                        CUSTOM_METRIC.append(custom_metric)
435                    if self.verbose > 0:
436                        if self.custom_metric is not None:
437                            print(
438                                {
439                                    "Model": name,
440                                    "Accuracy": accuracy,
441                                    "Balanced Accuracy": b_accuracy,
442                                    "ROC AUC": roc_auc,
443                                    "F1 Score": f1,
444                                    self.custom_metric.__name__: custom_metric,
445                                    "Time taken": time.time() - start,
446                                }
447                            )
448                        else:
449                            print(
450                                {
451                                    "Model": name,
452                                    "Accuracy": accuracy,
453                                    "Balanced Accuracy": b_accuracy,
454                                    "ROC AUC": roc_auc,
455                                    "F1 Score": f1,
456                                    "Time taken": time.time() - start,
457                                }
458                            )
459                    if self.predictions:
460                        predictions[name] = y_pred
461                except Exception as exception:
462                    if self.ignore_warnings is False:
463                        print(name + " model failed to execute")
464                        print(exception)
465
466        else:  # no preprocessing
467
468            for name, model in tqdm(self.classifiers):  # do parallel exec
469                start = time.time()
470                try:
471                    if "random_state" in model().get_params().keys():
472                        layer_clf = CustomClassifier(
473                            obj=model(random_state=self.random_state),
474                            n_hidden_features=self.n_hidden_features,
475                            activation_name=self.activation_name,
476                            a=self.a,
477                            nodes_sim=self.nodes_sim,
478                            bias=self.bias,
479                            dropout=self.dropout,
480                            direct_link=self.direct_link,
481                            n_clusters=self.n_clusters,
482                            cluster_encode=self.cluster_encode,
483                            type_clust=self.type_clust,
484                            type_scaling=self.type_scaling,
485                            col_sample=self.col_sample,
486                            row_sample=self.row_sample,
487                            seed=self.seed,
488                            backend=self.backend,
489                        )
490
491                    else:
492                        layer_clf = CustomClassifier(
493                            obj=model(),
494                            n_hidden_features=self.n_hidden_features,
495                            activation_name=self.activation_name,
496                            a=self.a,
497                            nodes_sim=self.nodes_sim,
498                            bias=self.bias,
499                            dropout=self.dropout,
500                            direct_link=self.direct_link,
501                            n_clusters=self.n_clusters,
502                            cluster_encode=self.cluster_encode,
503                            type_clust=self.type_clust,
504                            type_scaling=self.type_scaling,
505                            col_sample=self.col_sample,
506                            row_sample=self.row_sample,
507                            seed=self.seed,
508                            backend=self.backend,
509                        )
510
511                    layer_clf.fit(X_train, y_train)
512
513                    for _ in range(self.n_layers):
514                        layer_clf = deepcopy(
515                            CustomClassifier(
516                                obj=layer_clf,
517                                n_hidden_features=self.n_hidden_features,
518                                activation_name=self.activation_name,
519                                a=self.a,
520                                nodes_sim=self.nodes_sim,
521                                bias=self.bias,
522                                dropout=self.dropout,
523                                direct_link=self.direct_link,
524                                n_clusters=self.n_clusters,
525                                cluster_encode=self.cluster_encode,
526                                type_clust=self.type_clust,
527                                type_scaling=self.type_scaling,
528                                col_sample=self.col_sample,
529                                row_sample=self.row_sample,
530                                seed=self.seed,
531                                backend=self.backend,
532                            )
533                        )
534
535                        # layer_clf.fit(X_train, y_train)
536
537                    layer_clf.fit(X_train, y_train)
538
539                    self.models[name] = layer_clf
540                    y_pred = layer_clf.predict(X_test)
541                    accuracy = accuracy_score(y_test, y_pred, normalize=True)
542                    b_accuracy = balanced_accuracy_score(y_test, y_pred)
543                    f1 = f1_score(y_test, y_pred, average="weighted")
544                    try:
545                        roc_auc = roc_auc_score(y_test, y_pred)
546                    except Exception as exception:
547                        roc_auc = None
548                        if self.ignore_warnings is False:
549                            print("ROC AUC couldn't be calculated for " + name)
550                            print(exception)
551                    names.append(name)
552                    Accuracy.append(accuracy)
553                    B_Accuracy.append(b_accuracy)
554                    ROC_AUC.append(roc_auc)
555                    F1.append(f1)
556                    TIME.append(time.time() - start)
557                    if self.custom_metric is not None:
558                        custom_metric = self.custom_metric(y_test, y_pred)
559                        CUSTOM_METRIC.append(custom_metric)
560                    if self.verbose > 0:
561                        if self.custom_metric is not None:
562                            print(
563                                {
564                                    "Model": name,
565                                    "Accuracy": accuracy,
566                                    "Balanced Accuracy": b_accuracy,
567                                    "ROC AUC": roc_auc,
568                                    "F1 Score": f1,
569                                    self.custom_metric.__name__: custom_metric,
570                                    "Time taken": time.time() - start,
571                                }
572                            )
573                        else:
574                            print(
575                                {
576                                    "Model": name,
577                                    "Accuracy": accuracy,
578                                    "Balanced Accuracy": b_accuracy,
579                                    "ROC AUC": roc_auc,
580                                    "F1 Score": f1,
581                                    "Time taken": time.time() - start,
582                                }
583                            )
584                    if self.predictions:
585                        predictions[name] = y_pred
586                except Exception as exception:
587                    if self.ignore_warnings is False:
588                        print(name + " model failed to execute")
589                        print(exception)
590
591        if self.custom_metric is None:
592            scores = pd.DataFrame(
593                {
594                    "Model": names,
595                    "Accuracy": Accuracy,
596                    "Balanced Accuracy": B_Accuracy,
597                    "ROC AUC": ROC_AUC,
598                    "F1 Score": F1,
599                    "Time Taken": TIME,
600                }
601            )
602        else:
603            scores = pd.DataFrame(
604                {
605                    "Model": names,
606                    "Accuracy": Accuracy,
607                    "Balanced Accuracy": B_Accuracy,
608                    "ROC AUC": ROC_AUC,
609                    "F1 Score": F1,
610                    self.custom_metric.__name__: CUSTOM_METRIC,
611                    "Time Taken": TIME,
612                }
613            )
614        scores = scores.sort_values(by=self.sort_by, ascending=False).set_index(
615            "Model"
616        )
617
618        if self.predictions:
619            predictions_df = pd.DataFrame.from_dict(predictions)
620        return scores, predictions_df if self.predictions is True else scores
621
622    def provide_models(self, X_train, X_test, y_train, y_test):
623        """Returns all the model objects trained. If fit hasn't been called yet,
624        then it's called to return the models.
625
626        Parameters:
627
628        X_train: array-like,
629            Training vectors, where rows is the number of samples
630            and columns is the number of features.
631
632        X_test: array-like,
633            Testing vectors, where rows is the number of samples
634            and columns is the number of features.
635
636        y_train: array-like,
637            Training vectors, where rows is the number of samples
638            and columns is the number of features.
639
640        y_test: array-like,
641            Testing vectors, where rows is the number of samples
642            and columns is the number of features.
643
644        Returns:
645
646            models: dict-object,
647                Returns a dictionary with each model's pipeline as value
648                and key = name of the model.
649        """
650        if len(self.models.keys()) == 0:
651            self.fit(X_train, X_test, y_train, y_test)
652
653        return self.models

Fitting -- almost -- all the classification algorithms with layers of nnetsauce's CustomClassifier and returning their scores.

Parameters:

verbose: int, optional (default=0)
    Any positive number for verbosity.

ignore_warnings: bool, optional (default=True)
    When set to True, the warning related to algorigms that are not
    able to run are ignored.

custom_metric: function, optional (default=None)
    When function is provided, models are evaluated based on the custom
      evaluation metric provided.

predictions: bool, optional (default=False)
    When set to True, the predictions of all the models models are
    returned as data frame.

sort_by: string, optional (default='Accuracy')
    Sort models by a metric. Available options are 'Accuracy',
    'Balanced Accuracy', 'ROC AUC', 'F1 Score' or a custom metric
    identified by its name and provided by custom_metric.

random_state: int, optional (default=42)
    Reproducibiility seed.

estimators: list, optional (default='all')
    list of Estimators names or just 'all' for > 90 classifiers
    (default='all')

preprocess: bool, preprocessing is done when set to True

n_jobs: int, when possible, run in parallel
    For now, only used by individual models that support it.

n_layers: int, optional (default=3)
    Number of layers of CustomClassifiers to be used.

All the other parameters are the same as CustomClassifier's.

Examples

import nnetsauce as ns
    from sklearn.datasets import load_breast_cancer
    from sklearn.model_selection import train_test_split
    data = load_breast_cancer()
    X = data.data
    y= data.target
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2,
        random_state=123)
    clf = ns.LazyDeepClassifier(verbose=0, ignore_warnings=True, custom_metric=None)
    models, predictions = clf.fit(X_train, X_test, y_train, y_test)
    model_dictionary = clf.provide_models(X_train,X_test,y_train,y_test)
    print(models)
    

def fit(self, X_train, X_test, y_train, y_test):
210    def fit(self, X_train, X_test, y_train, y_test):
211        """Fit classifiers to X_train and y_train, predict and score on X_test,
212        y_test.
213
214        Parameters:
215
216            X_train: array-like,
217                Training vectors, where rows is the number of samples
218                and columns is the number of features.
219
220            X_test: array-like,
221                Testing vectors, where rows is the number of samples
222                and columns is the number of features.
223
224            y_train: array-like,
225                Training vectors, where rows is the number of samples
226                and columns is the number of features.
227
228            y_test: array-like,
229                Testing vectors, where rows is the number of samples
230                and columns is the number of features.
231
232        Returns:
233
234            scores: Pandas DataFrame
235                Returns metrics of all the models in a Pandas DataFrame.
236
237            predictions: Pandas DataFrame
238                Returns predictions of all the models in a Pandas DataFrame.
239        """
240        Accuracy = []
241        B_Accuracy = []
242        ROC_AUC = []
243        F1 = []
244        names = []
245        TIME = []
246        predictions = {}
247
248        if self.custom_metric is not None:
249            CUSTOM_METRIC = []
250
251        if isinstance(X_train, np.ndarray):
252            X_train = pd.DataFrame(X_train)
253            X_test = pd.DataFrame(X_test)
254
255        numeric_features = X_train.select_dtypes(include=[np.number]).columns
256        categorical_features = X_train.select_dtypes(include=["object"]).columns
257
258        categorical_low, categorical_high = get_card_split(
259            X_train, categorical_features
260        )
261
262        if self.preprocess is True:
263            preprocessor = ColumnTransformer(
264                transformers=[
265                    ("numeric", numeric_transformer, numeric_features),
266                    (
267                        "categorical_low",
268                        categorical_transformer_low,
269                        categorical_low,
270                    ),
271                    (
272                        "categorical_high",
273                        categorical_transformer_high,
274                        categorical_high,
275                    ),
276                ]
277            )
278
279        if self.estimators == "all":
280            self.classifiers = [
281                item
282                for sublist in [
283                    DEEPCLASSIFIERS,
284                    DEEPMULTITASKCLASSIFIERS,
285                    DEEPSIMPLEMULTITASKCLASSIFIERS,
286                ]
287                for item in sublist
288            ]
289        else:
290            self.classifiers = (
291                [
292                    ("DeepCustomClassifier(" + est[0] + ")", est[1])
293                    for est in all_estimators()
294                    if (
295                        issubclass(est[1], ClassifierMixin)
296                        and (est[0] in self.estimators)
297                    )
298                ]
299                + [
300                    (
301                        "DeepMultitaskClassifier(" + est[0] + ")",
302                        partial(MultitaskClassifier, obj=est[1]()),
303                    )
304                    for est in all_estimators()
305                    if (
306                        issubclass(est[1], RegressorMixin)
307                        and (est[0] in self.estimators)
308                    )
309                ]
310                + [
311                    (
312                        "DeepSimpleMultitaskClassifier(" + est[0] + ")",
313                        partial(SimpleMultitaskClassifier, obj=est[1]()),
314                    )
315                    for est in all_estimators()
316                    if (
317                        issubclass(est[1], RegressorMixin)
318                        and (est[0] in self.estimators)
319                    )
320                ]
321            )
322
323        if self.preprocess is True:
324
325            for name, model in tqdm(self.classifiers):  # do parallel exec
326
327                other_args = (
328                    {}
329                )  # use this trick for `random_state` too --> refactor
330                try:
331                    if (
332                        "n_jobs" in model().get_params().keys()
333                        and name.find("LogisticRegression") == -1
334                    ):
335                        other_args["n_jobs"] = self.n_jobs
336                except Exception:
337                    pass
338
339                start = time.time()
340
341                try:
342                    if "random_state" in model().get_params().keys():
343                        layer_clf = CustomClassifier(
344                            obj=model(random_state=self.random_state),
345                            n_hidden_features=self.n_hidden_features,
346                            activation_name=self.activation_name,
347                            a=self.a,
348                            nodes_sim=self.nodes_sim,
349                            bias=self.bias,
350                            dropout=self.dropout,
351                            direct_link=self.direct_link,
352                            n_clusters=self.n_clusters,
353                            cluster_encode=self.cluster_encode,
354                            type_clust=self.type_clust,
355                            type_scaling=self.type_scaling,
356                            col_sample=self.col_sample,
357                            row_sample=self.row_sample,
358                            seed=self.seed,
359                            backend=self.backend,
360                        )
361
362                    else:
363                        layer_clf = CustomClassifier(
364                            obj=model(),
365                            n_hidden_features=self.n_hidden_features,
366                            activation_name=self.activation_name,
367                            a=self.a,
368                            nodes_sim=self.nodes_sim,
369                            bias=self.bias,
370                            dropout=self.dropout,
371                            direct_link=self.direct_link,
372                            n_clusters=self.n_clusters,
373                            cluster_encode=self.cluster_encode,
374                            type_clust=self.type_clust,
375                            type_scaling=self.type_scaling,
376                            col_sample=self.col_sample,
377                            row_sample=self.row_sample,
378                            seed=self.seed,
379                            backend=self.backend,
380                        )
381
382                    layer_clf.fit(X_train, y_train)
383
384                    for _ in range(self.n_layers):
385                        layer_clf = deepcopy(
386                            CustomClassifier(
387                                obj=layer_clf,
388                                n_hidden_features=self.n_hidden_features,
389                                activation_name=self.activation_name,
390                                a=self.a,
391                                nodes_sim=self.nodes_sim,
392                                bias=self.bias,
393                                dropout=self.dropout,
394                                direct_link=self.direct_link,
395                                n_clusters=self.n_clusters,
396                                cluster_encode=self.cluster_encode,
397                                type_clust=self.type_clust,
398                                type_scaling=self.type_scaling,
399                                col_sample=self.col_sample,
400                                row_sample=self.row_sample,
401                                seed=self.seed,
402                                backend=self.backend,
403                            )
404                        )
405
406                    pipe = Pipeline(
407                        [
408                            ("preprocessor", preprocessor),
409                            ("classifier", layer_clf),
410                        ]
411                    )
412
413                    pipe.fit(X_train, y_train)
414                    self.models[name] = pipe
415                    y_pred = pipe.predict(X_test)
416                    accuracy = accuracy_score(y_test, y_pred, normalize=True)
417                    b_accuracy = balanced_accuracy_score(y_test, y_pred)
418                    f1 = f1_score(y_test, y_pred, average="weighted")
419                    try:
420                        roc_auc = roc_auc_score(y_test, y_pred)
421                    except Exception as exception:
422                        roc_auc = None
423                        if self.ignore_warnings is False:
424                            print("ROC AUC couldn't be calculated for " + name)
425                            print(exception)
426                    names.append(name)
427                    Accuracy.append(accuracy)
428                    B_Accuracy.append(b_accuracy)
429                    ROC_AUC.append(roc_auc)
430                    F1.append(f1)
431                    TIME.append(time.time() - start)
432                    if self.custom_metric is not None:
433                        custom_metric = self.custom_metric(y_test, y_pred)
434                        CUSTOM_METRIC.append(custom_metric)
435                    if self.verbose > 0:
436                        if self.custom_metric is not None:
437                            print(
438                                {
439                                    "Model": name,
440                                    "Accuracy": accuracy,
441                                    "Balanced Accuracy": b_accuracy,
442                                    "ROC AUC": roc_auc,
443                                    "F1 Score": f1,
444                                    self.custom_metric.__name__: custom_metric,
445                                    "Time taken": time.time() - start,
446                                }
447                            )
448                        else:
449                            print(
450                                {
451                                    "Model": name,
452                                    "Accuracy": accuracy,
453                                    "Balanced Accuracy": b_accuracy,
454                                    "ROC AUC": roc_auc,
455                                    "F1 Score": f1,
456                                    "Time taken": time.time() - start,
457                                }
458                            )
459                    if self.predictions:
460                        predictions[name] = y_pred
461                except Exception as exception:
462                    if self.ignore_warnings is False:
463                        print(name + " model failed to execute")
464                        print(exception)
465
466        else:  # no preprocessing
467
468            for name, model in tqdm(self.classifiers):  # do parallel exec
469                start = time.time()
470                try:
471                    if "random_state" in model().get_params().keys():
472                        layer_clf = CustomClassifier(
473                            obj=model(random_state=self.random_state),
474                            n_hidden_features=self.n_hidden_features,
475                            activation_name=self.activation_name,
476                            a=self.a,
477                            nodes_sim=self.nodes_sim,
478                            bias=self.bias,
479                            dropout=self.dropout,
480                            direct_link=self.direct_link,
481                            n_clusters=self.n_clusters,
482                            cluster_encode=self.cluster_encode,
483                            type_clust=self.type_clust,
484                            type_scaling=self.type_scaling,
485                            col_sample=self.col_sample,
486                            row_sample=self.row_sample,
487                            seed=self.seed,
488                            backend=self.backend,
489                        )
490
491                    else:
492                        layer_clf = CustomClassifier(
493                            obj=model(),
494                            n_hidden_features=self.n_hidden_features,
495                            activation_name=self.activation_name,
496                            a=self.a,
497                            nodes_sim=self.nodes_sim,
498                            bias=self.bias,
499                            dropout=self.dropout,
500                            direct_link=self.direct_link,
501                            n_clusters=self.n_clusters,
502                            cluster_encode=self.cluster_encode,
503                            type_clust=self.type_clust,
504                            type_scaling=self.type_scaling,
505                            col_sample=self.col_sample,
506                            row_sample=self.row_sample,
507                            seed=self.seed,
508                            backend=self.backend,
509                        )
510
511                    layer_clf.fit(X_train, y_train)
512
513                    for _ in range(self.n_layers):
514                        layer_clf = deepcopy(
515                            CustomClassifier(
516                                obj=layer_clf,
517                                n_hidden_features=self.n_hidden_features,
518                                activation_name=self.activation_name,
519                                a=self.a,
520                                nodes_sim=self.nodes_sim,
521                                bias=self.bias,
522                                dropout=self.dropout,
523                                direct_link=self.direct_link,
524                                n_clusters=self.n_clusters,
525                                cluster_encode=self.cluster_encode,
526                                type_clust=self.type_clust,
527                                type_scaling=self.type_scaling,
528                                col_sample=self.col_sample,
529                                row_sample=self.row_sample,
530                                seed=self.seed,
531                                backend=self.backend,
532                            )
533                        )
534
535                        # layer_clf.fit(X_train, y_train)
536
537                    layer_clf.fit(X_train, y_train)
538
539                    self.models[name] = layer_clf
540                    y_pred = layer_clf.predict(X_test)
541                    accuracy = accuracy_score(y_test, y_pred, normalize=True)
542                    b_accuracy = balanced_accuracy_score(y_test, y_pred)
543                    f1 = f1_score(y_test, y_pred, average="weighted")
544                    try:
545                        roc_auc = roc_auc_score(y_test, y_pred)
546                    except Exception as exception:
547                        roc_auc = None
548                        if self.ignore_warnings is False:
549                            print("ROC AUC couldn't be calculated for " + name)
550                            print(exception)
551                    names.append(name)
552                    Accuracy.append(accuracy)
553                    B_Accuracy.append(b_accuracy)
554                    ROC_AUC.append(roc_auc)
555                    F1.append(f1)
556                    TIME.append(time.time() - start)
557                    if self.custom_metric is not None:
558                        custom_metric = self.custom_metric(y_test, y_pred)
559                        CUSTOM_METRIC.append(custom_metric)
560                    if self.verbose > 0:
561                        if self.custom_metric is not None:
562                            print(
563                                {
564                                    "Model": name,
565                                    "Accuracy": accuracy,
566                                    "Balanced Accuracy": b_accuracy,
567                                    "ROC AUC": roc_auc,
568                                    "F1 Score": f1,
569                                    self.custom_metric.__name__: custom_metric,
570                                    "Time taken": time.time() - start,
571                                }
572                            )
573                        else:
574                            print(
575                                {
576                                    "Model": name,
577                                    "Accuracy": accuracy,
578                                    "Balanced Accuracy": b_accuracy,
579                                    "ROC AUC": roc_auc,
580                                    "F1 Score": f1,
581                                    "Time taken": time.time() - start,
582                                }
583                            )
584                    if self.predictions:
585                        predictions[name] = y_pred
586                except Exception as exception:
587                    if self.ignore_warnings is False:
588                        print(name + " model failed to execute")
589                        print(exception)
590
591        if self.custom_metric is None:
592            scores = pd.DataFrame(
593                {
594                    "Model": names,
595                    "Accuracy": Accuracy,
596                    "Balanced Accuracy": B_Accuracy,
597                    "ROC AUC": ROC_AUC,
598                    "F1 Score": F1,
599                    "Time Taken": TIME,
600                }
601            )
602        else:
603            scores = pd.DataFrame(
604                {
605                    "Model": names,
606                    "Accuracy": Accuracy,
607                    "Balanced Accuracy": B_Accuracy,
608                    "ROC AUC": ROC_AUC,
609                    "F1 Score": F1,
610                    self.custom_metric.__name__: CUSTOM_METRIC,
611                    "Time Taken": TIME,
612                }
613            )
614        scores = scores.sort_values(by=self.sort_by, ascending=False).set_index(
615            "Model"
616        )
617
618        if self.predictions:
619            predictions_df = pd.DataFrame.from_dict(predictions)
620        return scores, predictions_df if self.predictions is True else scores

Fit classifiers to X_train and y_train, predict and score on X_test, y_test.

Parameters:

X_train: array-like,
    Training vectors, where rows is the number of samples
    and columns is the number of features.

X_test: array-like,
    Testing vectors, where rows is the number of samples
    and columns is the number of features.

y_train: array-like,
    Training vectors, where rows is the number of samples
    and columns is the number of features.

y_test: array-like,
    Testing vectors, where rows is the number of samples
    and columns is the number of features.

Returns:

scores: Pandas DataFrame
    Returns metrics of all the models in a Pandas DataFrame.

predictions: Pandas DataFrame
    Returns predictions of all the models in a Pandas DataFrame.
def provide_models(self, X_train, X_test, y_train, y_test):
622    def provide_models(self, X_train, X_test, y_train, y_test):
623        """Returns all the model objects trained. If fit hasn't been called yet,
624        then it's called to return the models.
625
626        Parameters:
627
628        X_train: array-like,
629            Training vectors, where rows is the number of samples
630            and columns is the number of features.
631
632        X_test: array-like,
633            Testing vectors, where rows is the number of samples
634            and columns is the number of features.
635
636        y_train: array-like,
637            Training vectors, where rows is the number of samples
638            and columns is the number of features.
639
640        y_test: array-like,
641            Testing vectors, where rows is the number of samples
642            and columns is the number of features.
643
644        Returns:
645
646            models: dict-object,
647                Returns a dictionary with each model's pipeline as value
648                and key = name of the model.
649        """
650        if len(self.models.keys()) == 0:
651            self.fit(X_train, X_test, y_train, y_test)
652
653        return self.models

Returns all the model objects trained. If fit hasn't been called yet, then it's called to return the models.

Parameters:

X_train: array-like, Training vectors, where rows is the number of samples and columns is the number of features.

X_test: array-like, Testing vectors, where rows is the number of samples and columns is the number of features.

y_train: array-like, Training vectors, where rows is the number of samples and columns is the number of features.

y_test: array-like, Testing vectors, where rows is the number of samples and columns is the number of features.

Returns:

models: dict-object,
    Returns a dictionary with each model's pipeline as value
    and key = name of the model.
class LazyDeepRegressor(nnetsauce.custom.custom.Custom, sklearn.base.RegressorMixin):
 83class LazyDeepRegressor(Custom, RegressorMixin):
 84    """
 85        Fitting -- almost -- all the regression algorithms with layers of
 86        nnetsauce's CustomRegressor and returning their scores.
 87
 88    Parameters:
 89
 90        verbose: int, optional (default=0)
 91            Any positive number for verbosity.
 92
 93        ignore_warnings: bool, optional (default=True)
 94            When set to True, the warning related to algorigms that are not able to run are ignored.
 95
 96        custom_metric: function, optional (default=None)
 97            When function is provided, models are evaluated based on the custom evaluation metric provided.
 98
 99        predictions: bool, optional (default=False)
100            When set to True, the predictions of all the models models are returned as dataframe.
101
102        sort_by: string, optional (default='Accuracy')
103            Sort models by a metric. Available options are 'Accuracy', 'Balanced Accuracy', 'ROC AUC', 'F1 Score'
104            or a custom metric identified by its name and provided by custom_metric.
105
106        random_state: int, optional (default=42)
107            Reproducibiility seed.
108
109        estimators: list, optional (default='all')
110            list of Estimators names or just 'all' (default='all')
111
112        preprocess: bool
113            preprocessing is done when set to True
114
115        n_jobs : int, when possible, run in parallel
116            For now, only used by individual models that support it.
117
118        n_layers: int, optional (default=3)
119            Number of layers of CustomRegressors to be used.
120
121        All the other parameters are the same as CustomRegressor's.
122
123    Examples:
124
125        import nnetsauce as ns
126        import numpy as np
127        from sklearn import datasets
128        from sklearn.utils import shuffle
129
130        diabetes = datasets.load_diabetes()
131        X, y = shuffle(diabetes.data, diabetes.target, random_state=13)
132        X = X.astype(np.float32)
133
134        offset = int(X.shape[0] * 0.9)
135        X_train, y_train = X[:offset], y[:offset]
136        X_test, y_test = X[offset:], y[offset:]
137
138        reg = ns.LazyDeepRegressor(verbose=0, ignore_warnings=False, custom_metric=None)
139        models, predictions = reg.fit(X_train, X_test, y_train, y_test)
140        print(models)
141
142    """
143
144    def __init__(
145        self,
146        verbose=0,
147        ignore_warnings=True,
148        custom_metric=None,
149        predictions=False,
150        random_state=42,
151        estimators="all",
152        preprocess=False,
153        n_jobs=None,
154        # Defining depth
155        n_layers=3,
156        # CustomRegressor attributes
157        obj=None,
158        n_hidden_features=5,
159        activation_name="relu",
160        a=0.01,
161        nodes_sim="sobol",
162        bias=True,
163        dropout=0,
164        direct_link=True,
165        n_clusters=2,
166        cluster_encode=True,
167        type_clust="kmeans",
168        type_scaling=("std", "std", "std"),
169        col_sample=1,
170        row_sample=1,
171        seed=123,
172        backend="cpu",
173    ):
174        self.verbose = verbose
175        self.ignore_warnings = ignore_warnings
176        self.custom_metric = custom_metric
177        self.predictions = predictions
178        self.models = {}
179        self.random_state = random_state
180        self.estimators = estimators
181        self.preprocess = preprocess
182        self.n_layers = n_layers - 1
183        self.n_jobs = n_jobs
184        super().__init__(
185            obj=obj,
186            n_hidden_features=n_hidden_features,
187            activation_name=activation_name,
188            a=a,
189            nodes_sim=nodes_sim,
190            bias=bias,
191            dropout=dropout,
192            direct_link=direct_link,
193            n_clusters=n_clusters,
194            cluster_encode=cluster_encode,
195            type_clust=type_clust,
196            type_scaling=type_scaling,
197            col_sample=col_sample,
198            row_sample=row_sample,
199            seed=seed,
200            backend=backend,
201        )
202
203    def fit(self, X_train, X_test, y_train, y_test):
204        """Fit Regression algorithms to X_train and y_train, predict and score on X_test, y_test.
205
206        Parameters:
207
208            X_train : array-like,
209                Training vectors, where rows is the number of samples
210                and columns is the number of features.
211
212            X_test : array-like,
213                Testing vectors, where rows is the number of samples
214                and columns is the number of features.
215
216            y_train : array-like,
217                Training vectors, where rows is the number of samples
218                and columns is the number of features.
219
220            y_test : array-like,
221                Testing vectors, where rows is the number of samples
222                and columns is the number of features.
223
224        Returns:
225        -------
226        scores:  Pandas DataFrame
227            Returns metrics of all the models in a Pandas DataFrame.
228
229        predictions : Pandas DataFrame
230            Returns predictions of all the models in a Pandas DataFrame.
231
232        """
233        R2 = []
234        ADJR2 = []
235        RMSE = []
236        # WIN = []
237        names = []
238        TIME = []
239        predictions = {}
240
241        if self.custom_metric:
242            CUSTOM_METRIC = []
243
244        if isinstance(X_train, np.ndarray):
245            X_train = pd.DataFrame(X_train)
246            X_test = pd.DataFrame(X_test)
247
248        numeric_features = X_train.select_dtypes(include=[np.number]).columns
249        categorical_features = X_train.select_dtypes(include=["object"]).columns
250
251        categorical_low, categorical_high = get_card_split(
252            X_train, categorical_features
253        )
254
255        if self.preprocess is True:
256            preprocessor = ColumnTransformer(
257                transformers=[
258                    ("numeric", numeric_transformer, numeric_features),
259                    (
260                        "categorical_low",
261                        categorical_transformer_low,
262                        categorical_low,
263                    ),
264                    (
265                        "categorical_high",
266                        categorical_transformer_high,
267                        categorical_high,
268                    ),
269                ]
270            )
271
272        if self.estimators == "all":
273            self.regressors = DEEPREGRESSORS
274        else:
275            self.regressors = [
276                ("DeepCustomRegressor(" + est[0] + ")", est[1])
277                for est in all_estimators()
278                if (
279                    issubclass(est[1], RegressorMixin)
280                    and (est[0] in self.estimators)
281                )
282            ]
283
284        if self.preprocess is True:
285
286            for name, model in tqdm(self.regressors):  # do parallel exec
287                start = time.time()
288                try:
289                    if "random_state" in model().get_params().keys():
290                        layer_regr = CustomRegressor(
291                            obj=model(random_state=self.random_state),
292                            n_hidden_features=self.n_hidden_features,
293                            activation_name=self.activation_name,
294                            a=self.a,
295                            nodes_sim=self.nodes_sim,
296                            bias=self.bias,
297                            dropout=self.dropout,
298                            direct_link=self.direct_link,
299                            n_clusters=self.n_clusters,
300                            cluster_encode=self.cluster_encode,
301                            type_clust=self.type_clust,
302                            type_scaling=self.type_scaling,
303                            col_sample=self.col_sample,
304                            row_sample=self.row_sample,
305                            seed=self.seed,
306                            backend=self.backend,
307                        )
308                    else:
309                        layer_regr = CustomRegressor(
310                            obj=model(),
311                            n_hidden_features=self.n_hidden_features,
312                            activation_name=self.activation_name,
313                            a=self.a,
314                            nodes_sim=self.nodes_sim,
315                            bias=self.bias,
316                            dropout=self.dropout,
317                            direct_link=self.direct_link,
318                            n_clusters=self.n_clusters,
319                            cluster_encode=self.cluster_encode,
320                            type_clust=self.type_clust,
321                            type_scaling=self.type_scaling,
322                            col_sample=self.col_sample,
323                            row_sample=self.row_sample,
324                            seed=self.seed,
325                            backend=self.backend,
326                        )
327
328                    for _ in range(self.n_layers):
329                        layer_regr = deepcopy(
330                            CustomRegressor(
331                                obj=layer_regr,
332                                n_hidden_features=self.n_hidden_features,
333                                activation_name=self.activation_name,
334                                a=self.a,
335                                nodes_sim=self.nodes_sim,
336                                bias=self.bias,
337                                dropout=self.dropout,
338                                direct_link=self.direct_link,
339                                n_clusters=self.n_clusters,
340                                cluster_encode=self.cluster_encode,
341                                type_clust=self.type_clust,
342                                type_scaling=self.type_scaling,
343                                col_sample=self.col_sample,
344                                row_sample=self.row_sample,
345                                seed=self.seed,
346                                backend=self.backend,
347                            )
348                        )
349
350                    layer_regr.fit(X_train, y_train)
351
352                    pipe = Pipeline(
353                        steps=[
354                            ("preprocessor", preprocessor),
355                            ("regressor", layer_regr),
356                        ]
357                    )
358
359                    pipe.fit(X_train, y_train)
360
361                    self.models[name] = pipe
362                    y_pred = pipe.predict(X_test)
363                    r_squared = r2_score(y_test, y_pred)
364                    adj_rsquared = adjusted_rsquared(
365                        r_squared, X_test.shape[0], X_test.shape[1]
366                    )
367                    rmse = mean_squared_error(y_test, y_pred, squared=False)
368
369                    names.append(name)
370                    R2.append(r_squared)
371                    ADJR2.append(adj_rsquared)
372                    RMSE.append(rmse)
373                    TIME.append(time.time() - start)
374
375                    if self.custom_metric:
376                        custom_metric = self.custom_metric(y_test, y_pred)
377                        CUSTOM_METRIC.append(custom_metric)
378
379                    if self.verbose > 0:
380                        scores_verbose = {
381                            "Model": name,
382                            "R-Squared": r_squared,
383                            "Adjusted R-Squared": adj_rsquared,
384                            "RMSE": rmse,
385                            "Time taken": time.time() - start,
386                        }
387
388                        if self.custom_metric:
389                            scores_verbose[self.custom_metric.__name__] = (
390                                custom_metric
391                            )
392
393                        print(scores_verbose)
394                    if self.predictions:
395                        predictions[name] = y_pred
396                except Exception as exception:
397                    if self.ignore_warnings is False:
398                        print(name + " model failed to execute")
399                        print(exception)
400
401        else:  # no preprocessing
402
403            for name, model in tqdm(self.regressors):  # do parallel exec
404                start = time.time()
405                try:
406                    if "random_state" in model().get_params().keys():
407                        layer_regr = CustomRegressor(
408                            obj=model(random_state=self.random_state),
409                            n_hidden_features=self.n_hidden_features,
410                            activation_name=self.activation_name,
411                            a=self.a,
412                            nodes_sim=self.nodes_sim,
413                            bias=self.bias,
414                            dropout=self.dropout,
415                            direct_link=self.direct_link,
416                            n_clusters=self.n_clusters,
417                            cluster_encode=self.cluster_encode,
418                            type_clust=self.type_clust,
419                            type_scaling=self.type_scaling,
420                            col_sample=self.col_sample,
421                            row_sample=self.row_sample,
422                            seed=self.seed,
423                            backend=self.backend,
424                        )
425                    else:
426                        layer_regr = CustomRegressor(
427                            obj=model(),
428                            n_hidden_features=self.n_hidden_features,
429                            activation_name=self.activation_name,
430                            a=self.a,
431                            nodes_sim=self.nodes_sim,
432                            bias=self.bias,
433                            dropout=self.dropout,
434                            direct_link=self.direct_link,
435                            n_clusters=self.n_clusters,
436                            cluster_encode=self.cluster_encode,
437                            type_clust=self.type_clust,
438                            type_scaling=self.type_scaling,
439                            col_sample=self.col_sample,
440                            row_sample=self.row_sample,
441                            seed=self.seed,
442                            backend=self.backend,
443                        )
444
445                    layer_regr.fit(X_train, y_train)
446
447                    for _ in range(self.n_layers):
448                        layer_regr = deepcopy(
449                            CustomRegressor(
450                                obj=layer_regr,
451                                n_hidden_features=self.n_hidden_features,
452                                activation_name=self.activation_name,
453                                a=self.a,
454                                nodes_sim=self.nodes_sim,
455                                bias=self.bias,
456                                dropout=self.dropout,
457                                direct_link=self.direct_link,
458                                n_clusters=self.n_clusters,
459                                cluster_encode=self.cluster_encode,
460                                type_clust=self.type_clust,
461                                type_scaling=self.type_scaling,
462                                col_sample=self.col_sample,
463                                row_sample=self.row_sample,
464                                seed=self.seed,
465                                backend=self.backend,
466                            )
467                        )
468
469                        # layer_regr.fit(X_train, y_train)
470
471                    layer_regr.fit(X_train, y_train)
472
473                    self.models[name] = layer_regr
474                    y_pred = layer_regr.predict(X_test)
475
476                    r_squared = r2_score(y_test, y_pred)
477                    adj_rsquared = adjusted_rsquared(
478                        r_squared, X_test.shape[0], X_test.shape[1]
479                    )
480                    rmse = mean_squared_error(y_test, y_pred, squared=False)
481
482                    names.append(name)
483                    R2.append(r_squared)
484                    ADJR2.append(adj_rsquared)
485                    RMSE.append(rmse)
486                    TIME.append(time.time() - start)
487
488                    if self.custom_metric:
489                        custom_metric = self.custom_metric(y_test, y_pred)
490                        CUSTOM_METRIC.append(custom_metric)
491
492                    if self.verbose > 0:
493                        scores_verbose = {
494                            "Model": name,
495                            "R-Squared": r_squared,
496                            "Adjusted R-Squared": adj_rsquared,
497                            "RMSE": rmse,
498                            "Time taken": time.time() - start,
499                        }
500
501                        if self.custom_metric:
502                            scores_verbose[self.custom_metric.__name__] = (
503                                custom_metric
504                            )
505
506                        print(scores_verbose)
507                    if self.predictions:
508                        predictions[name] = y_pred
509                except Exception as exception:
510                    if self.ignore_warnings is False:
511                        print(name + " model failed to execute")
512                        print(exception)
513
514        scores = {
515            "Model": names,
516            "Adjusted R-Squared": ADJR2,
517            "R-Squared": R2,
518            "RMSE": RMSE,
519            "Time Taken": TIME,
520        }
521
522        if self.custom_metric:
523            scores[self.custom_metric.__name__] = CUSTOM_METRIC
524
525        scores = pd.DataFrame(scores)
526        scores = scores.sort_values(by="RMSE", ascending=True).set_index(
527            "Model"
528        )
529
530        if self.predictions:
531            predictions_df = pd.DataFrame.from_dict(predictions)
532        return scores, predictions_df if self.predictions is True else scores
533
534    def provide_models(self, X_train, X_test, y_train, y_test):
535        """
536        This function returns all the model objects trained in fit function.
537        If fit is not called already, then we call fit and then return the models.
538
539        Parameters:
540
541            X_train : array-like,
542                Training vectors, where rows is the number of samples
543                and columns is the number of features.
544
545            X_test : array-like,
546                Testing vectors, where rows is the number of samples
547                and columns is the number of features.
548
549            y_train : array-like,
550                Training vectors, where rows is the number of samples
551                and columns is the number of features.
552
553            y_test : array-like,
554                Testing vectors, where rows is the number of samples
555                and columns is the number of features.
556
557        Returns:
558
559            models: dict-object,
560                Returns a dictionary with each model pipeline as value
561                with key as name of models.
562
563        """
564        if len(self.models.keys()) == 0:
565            self.fit(X_train, X_test, y_train, y_test)
566
567        return self.models

Fitting -- almost -- all the regression algorithms with layers of nnetsauce's CustomRegressor and returning their scores.

Parameters:

verbose: int, optional (default=0)
    Any positive number for verbosity.

ignore_warnings: bool, optional (default=True)
    When set to True, the warning related to algorigms that are not able to run are ignored.

custom_metric: function, optional (default=None)
    When function is provided, models are evaluated based on the custom evaluation metric provided.

predictions: bool, optional (default=False)
    When set to True, the predictions of all the models models are returned as dataframe.

sort_by: string, optional (default='Accuracy')
    Sort models by a metric. Available options are 'Accuracy', 'Balanced Accuracy', 'ROC AUC', 'F1 Score'
    or a custom metric identified by its name and provided by custom_metric.

random_state: int, optional (default=42)
    Reproducibiility seed.

estimators: list, optional (default='all')
    list of Estimators names or just 'all' (default='all')

preprocess: bool
    preprocessing is done when set to True

n_jobs : int, when possible, run in parallel
    For now, only used by individual models that support it.

n_layers: int, optional (default=3)
    Number of layers of CustomRegressors to be used.

All the other parameters are the same as CustomRegressor's.

Examples:

import nnetsauce as ns
import numpy as np
from sklearn import datasets
from sklearn.utils import shuffle

diabetes = datasets.load_diabetes()
X, y = shuffle(diabetes.data, diabetes.target, random_state=13)
X = X.astype(np.float32)

offset = int(X.shape[0] * 0.9)
X_train, y_train = X[:offset], y[:offset]
X_test, y_test = X[offset:], y[offset:]

reg = ns.LazyDeepRegressor(verbose=0, ignore_warnings=False, custom_metric=None)
models, predictions = reg.fit(X_train, X_test, y_train, y_test)
print(models)
def fit(self, X_train, X_test, y_train, y_test):
203    def fit(self, X_train, X_test, y_train, y_test):
204        """Fit Regression algorithms to X_train and y_train, predict and score on X_test, y_test.
205
206        Parameters:
207
208            X_train : array-like,
209                Training vectors, where rows is the number of samples
210                and columns is the number of features.
211
212            X_test : array-like,
213                Testing vectors, where rows is the number of samples
214                and columns is the number of features.
215
216            y_train : array-like,
217                Training vectors, where rows is the number of samples
218                and columns is the number of features.
219
220            y_test : array-like,
221                Testing vectors, where rows is the number of samples
222                and columns is the number of features.
223
224        Returns:
225        -------
226        scores:  Pandas DataFrame
227            Returns metrics of all the models in a Pandas DataFrame.
228
229        predictions : Pandas DataFrame
230            Returns predictions of all the models in a Pandas DataFrame.
231
232        """
233        R2 = []
234        ADJR2 = []
235        RMSE = []
236        # WIN = []
237        names = []
238        TIME = []
239        predictions = {}
240
241        if self.custom_metric:
242            CUSTOM_METRIC = []
243
244        if isinstance(X_train, np.ndarray):
245            X_train = pd.DataFrame(X_train)
246            X_test = pd.DataFrame(X_test)
247
248        numeric_features = X_train.select_dtypes(include=[np.number]).columns
249        categorical_features = X_train.select_dtypes(include=["object"]).columns
250
251        categorical_low, categorical_high = get_card_split(
252            X_train, categorical_features
253        )
254
255        if self.preprocess is True:
256            preprocessor = ColumnTransformer(
257                transformers=[
258                    ("numeric", numeric_transformer, numeric_features),
259                    (
260                        "categorical_low",
261                        categorical_transformer_low,
262                        categorical_low,
263                    ),
264                    (
265                        "categorical_high",
266                        categorical_transformer_high,
267                        categorical_high,
268                    ),
269                ]
270            )
271
272        if self.estimators == "all":
273            self.regressors = DEEPREGRESSORS
274        else:
275            self.regressors = [
276                ("DeepCustomRegressor(" + est[0] + ")", est[1])
277                for est in all_estimators()
278                if (
279                    issubclass(est[1], RegressorMixin)
280                    and (est[0] in self.estimators)
281                )
282            ]
283
284        if self.preprocess is True:
285
286            for name, model in tqdm(self.regressors):  # do parallel exec
287                start = time.time()
288                try:
289                    if "random_state" in model().get_params().keys():
290                        layer_regr = CustomRegressor(
291                            obj=model(random_state=self.random_state),
292                            n_hidden_features=self.n_hidden_features,
293                            activation_name=self.activation_name,
294                            a=self.a,
295                            nodes_sim=self.nodes_sim,
296                            bias=self.bias,
297                            dropout=self.dropout,
298                            direct_link=self.direct_link,
299                            n_clusters=self.n_clusters,
300                            cluster_encode=self.cluster_encode,
301                            type_clust=self.type_clust,
302                            type_scaling=self.type_scaling,
303                            col_sample=self.col_sample,
304                            row_sample=self.row_sample,
305                            seed=self.seed,
306                            backend=self.backend,
307                        )
308                    else:
309                        layer_regr = CustomRegressor(
310                            obj=model(),
311                            n_hidden_features=self.n_hidden_features,
312                            activation_name=self.activation_name,
313                            a=self.a,
314                            nodes_sim=self.nodes_sim,
315                            bias=self.bias,
316                            dropout=self.dropout,
317                            direct_link=self.direct_link,
318                            n_clusters=self.n_clusters,
319                            cluster_encode=self.cluster_encode,
320                            type_clust=self.type_clust,
321                            type_scaling=self.type_scaling,
322                            col_sample=self.col_sample,
323                            row_sample=self.row_sample,
324                            seed=self.seed,
325                            backend=self.backend,
326                        )
327
328                    for _ in range(self.n_layers):
329                        layer_regr = deepcopy(
330                            CustomRegressor(
331                                obj=layer_regr,
332                                n_hidden_features=self.n_hidden_features,
333                                activation_name=self.activation_name,
334                                a=self.a,
335                                nodes_sim=self.nodes_sim,
336                                bias=self.bias,
337                                dropout=self.dropout,
338                                direct_link=self.direct_link,
339                                n_clusters=self.n_clusters,
340                                cluster_encode=self.cluster_encode,
341                                type_clust=self.type_clust,
342                                type_scaling=self.type_scaling,
343                                col_sample=self.col_sample,
344                                row_sample=self.row_sample,
345                                seed=self.seed,
346                                backend=self.backend,
347                            )
348                        )
349
350                    layer_regr.fit(X_train, y_train)
351
352                    pipe = Pipeline(
353                        steps=[
354                            ("preprocessor", preprocessor),
355                            ("regressor", layer_regr),
356                        ]
357                    )
358
359                    pipe.fit(X_train, y_train)
360
361                    self.models[name] = pipe
362                    y_pred = pipe.predict(X_test)
363                    r_squared = r2_score(y_test, y_pred)
364                    adj_rsquared = adjusted_rsquared(
365                        r_squared, X_test.shape[0], X_test.shape[1]
366                    )
367                    rmse = mean_squared_error(y_test, y_pred, squared=False)
368
369                    names.append(name)
370                    R2.append(r_squared)
371                    ADJR2.append(adj_rsquared)
372                    RMSE.append(rmse)
373                    TIME.append(time.time() - start)
374
375                    if self.custom_metric:
376                        custom_metric = self.custom_metric(y_test, y_pred)
377                        CUSTOM_METRIC.append(custom_metric)
378
379                    if self.verbose > 0:
380                        scores_verbose = {
381                            "Model": name,
382                            "R-Squared": r_squared,
383                            "Adjusted R-Squared": adj_rsquared,
384                            "RMSE": rmse,
385                            "Time taken": time.time() - start,
386                        }
387
388                        if self.custom_metric:
389                            scores_verbose[self.custom_metric.__name__] = (
390                                custom_metric
391                            )
392
393                        print(scores_verbose)
394                    if self.predictions:
395                        predictions[name] = y_pred
396                except Exception as exception:
397                    if self.ignore_warnings is False:
398                        print(name + " model failed to execute")
399                        print(exception)
400
401        else:  # no preprocessing
402
403            for name, model in tqdm(self.regressors):  # do parallel exec
404                start = time.time()
405                try:
406                    if "random_state" in model().get_params().keys():
407                        layer_regr = CustomRegressor(
408                            obj=model(random_state=self.random_state),
409                            n_hidden_features=self.n_hidden_features,
410                            activation_name=self.activation_name,
411                            a=self.a,
412                            nodes_sim=self.nodes_sim,
413                            bias=self.bias,
414                            dropout=self.dropout,
415                            direct_link=self.direct_link,
416                            n_clusters=self.n_clusters,
417                            cluster_encode=self.cluster_encode,
418                            type_clust=self.type_clust,
419                            type_scaling=self.type_scaling,
420                            col_sample=self.col_sample,
421                            row_sample=self.row_sample,
422                            seed=self.seed,
423                            backend=self.backend,
424                        )
425                    else:
426                        layer_regr = CustomRegressor(
427                            obj=model(),
428                            n_hidden_features=self.n_hidden_features,
429                            activation_name=self.activation_name,
430                            a=self.a,
431                            nodes_sim=self.nodes_sim,
432                            bias=self.bias,
433                            dropout=self.dropout,
434                            direct_link=self.direct_link,
435                            n_clusters=self.n_clusters,
436                            cluster_encode=self.cluster_encode,
437                            type_clust=self.type_clust,
438                            type_scaling=self.type_scaling,
439                            col_sample=self.col_sample,
440                            row_sample=self.row_sample,
441                            seed=self.seed,
442                            backend=self.backend,
443                        )
444
445                    layer_regr.fit(X_train, y_train)
446
447                    for _ in range(self.n_layers):
448                        layer_regr = deepcopy(
449                            CustomRegressor(
450                                obj=layer_regr,
451                                n_hidden_features=self.n_hidden_features,
452                                activation_name=self.activation_name,
453                                a=self.a,
454                                nodes_sim=self.nodes_sim,
455                                bias=self.bias,
456                                dropout=self.dropout,
457                                direct_link=self.direct_link,
458                                n_clusters=self.n_clusters,
459                                cluster_encode=self.cluster_encode,
460                                type_clust=self.type_clust,
461                                type_scaling=self.type_scaling,
462                                col_sample=self.col_sample,
463                                row_sample=self.row_sample,
464                                seed=self.seed,
465                                backend=self.backend,
466                            )
467                        )
468
469                        # layer_regr.fit(X_train, y_train)
470
471                    layer_regr.fit(X_train, y_train)
472
473                    self.models[name] = layer_regr
474                    y_pred = layer_regr.predict(X_test)
475
476                    r_squared = r2_score(y_test, y_pred)
477                    adj_rsquared = adjusted_rsquared(
478                        r_squared, X_test.shape[0], X_test.shape[1]
479                    )
480                    rmse = mean_squared_error(y_test, y_pred, squared=False)
481
482                    names.append(name)
483                    R2.append(r_squared)
484                    ADJR2.append(adj_rsquared)
485                    RMSE.append(rmse)
486                    TIME.append(time.time() - start)
487
488                    if self.custom_metric:
489                        custom_metric = self.custom_metric(y_test, y_pred)
490                        CUSTOM_METRIC.append(custom_metric)
491
492                    if self.verbose > 0:
493                        scores_verbose = {
494                            "Model": name,
495                            "R-Squared": r_squared,
496                            "Adjusted R-Squared": adj_rsquared,
497                            "RMSE": rmse,
498                            "Time taken": time.time() - start,
499                        }
500
501                        if self.custom_metric:
502                            scores_verbose[self.custom_metric.__name__] = (
503                                custom_metric
504                            )
505
506                        print(scores_verbose)
507                    if self.predictions:
508                        predictions[name] = y_pred
509                except Exception as exception:
510                    if self.ignore_warnings is False:
511                        print(name + " model failed to execute")
512                        print(exception)
513
514        scores = {
515            "Model": names,
516            "Adjusted R-Squared": ADJR2,
517            "R-Squared": R2,
518            "RMSE": RMSE,
519            "Time Taken": TIME,
520        }
521
522        if self.custom_metric:
523            scores[self.custom_metric.__name__] = CUSTOM_METRIC
524
525        scores = pd.DataFrame(scores)
526        scores = scores.sort_values(by="RMSE", ascending=True).set_index(
527            "Model"
528        )
529
530        if self.predictions:
531            predictions_df = pd.DataFrame.from_dict(predictions)
532        return scores, predictions_df if self.predictions is True else scores

Fit Regression algorithms to X_train and y_train, predict and score on X_test, y_test.

Parameters:

X_train : array-like,
    Training vectors, where rows is the number of samples
    and columns is the number of features.

X_test : array-like,
    Testing vectors, where rows is the number of samples
    and columns is the number of features.

y_train : array-like,
    Training vectors, where rows is the number of samples
    and columns is the number of features.

y_test : array-like,
    Testing vectors, where rows is the number of samples
    and columns is the number of features.

Returns:

scores: Pandas DataFrame Returns metrics of all the models in a Pandas DataFrame.

predictions : Pandas DataFrame Returns predictions of all the models in a Pandas DataFrame.

def provide_models(self, X_train, X_test, y_train, y_test):
534    def provide_models(self, X_train, X_test, y_train, y_test):
535        """
536        This function returns all the model objects trained in fit function.
537        If fit is not called already, then we call fit and then return the models.
538
539        Parameters:
540
541            X_train : array-like,
542                Training vectors, where rows is the number of samples
543                and columns is the number of features.
544
545            X_test : array-like,
546                Testing vectors, where rows is the number of samples
547                and columns is the number of features.
548
549            y_train : array-like,
550                Training vectors, where rows is the number of samples
551                and columns is the number of features.
552
553            y_test : array-like,
554                Testing vectors, where rows is the number of samples
555                and columns is the number of features.
556
557        Returns:
558
559            models: dict-object,
560                Returns a dictionary with each model pipeline as value
561                with key as name of models.
562
563        """
564        if len(self.models.keys()) == 0:
565            self.fit(X_train, X_test, y_train, y_test)
566
567        return self.models

This function returns all the model objects trained in fit function. If fit is not called already, then we call fit and then return the models.

Parameters:

X_train : array-like,
    Training vectors, where rows is the number of samples
    and columns is the number of features.

X_test : array-like,
    Testing vectors, where rows is the number of samples
    and columns is the number of features.

y_train : array-like,
    Training vectors, where rows is the number of samples
    and columns is the number of features.

y_test : array-like,
    Testing vectors, where rows is the number of samples
    and columns is the number of features.

Returns:

models: dict-object,
    Returns a dictionary with each model pipeline as value
    with key as name of models.
class LazyDeepMTS(nnetsauce.MTS):
 99class LazyDeepMTS(MTS):
100    """
101
102    Fitting -- almost -- all the regression algorithms with layers of
103    nnetsauce's CustomRegressor to multivariate time series
104    and returning their scores.
105
106    Parameters:
107
108        verbose: int, optional (default=0)
109            Any positive number for verbosity.
110
111        ignore_warnings: bool, optional (default=True)
112            When set to True, the warning related to algorigms that are not
113            able to run are ignored.
114
115        custom_metric: function, optional (default=None)
116            When function is provided, models are evaluated based on the custom
117              evaluation metric provided.
118
119        predictions: bool, optional (default=False)
120            When set to True, the predictions of all the models models are returned as dataframe.
121
122        sort_by: string, optional (default='RMSE')
123            Sort models by a metric. Available options are 'RMSE', 'MAE', 'MPL', 'MPE', 'MAPE',
124            'R-Squared', 'Adjusted R-Squared' or a custom metric identified by its name and
125            provided by custom_metric.
126
127        random_state: int, optional (default=42)
128            Reproducibiility seed.
129
130        estimators: list, optional (default='all')
131            list of Estimators (regression algorithms) names or just 'all' (default='all')
132
133        preprocess: bool, preprocessing is done when set to True
134
135        n_layers: int, optional (default=1)
136            Number of layers in the network. When set to 1, the model is equivalent to a MTS.
137
138        h: int, optional (default=None)
139            Number of steps ahead to predict (when used, must be > 0 and < X_test.shape[0]).
140
141        All the other parameters are the same as MTS's.
142
143    Examples:
144
145        See https://thierrymoudiki.github.io/blog/2023/10/29/python/quasirandomizednn/MTS-LazyPredict
146
147    """
148
149    def __init__(
150        self,
151        verbose=0,
152        ignore_warnings=True,
153        custom_metric=None,
154        predictions=False,
155        random_state=42,
156        estimators="all",
157        preprocess=False,
158        n_layers=1,
159        h=None,
160        # MTS attributes
161        obj=None,
162        n_hidden_features=5,
163        activation_name="relu",
164        a=0.01,
165        nodes_sim="sobol",
166        bias=True,
167        dropout=0,
168        direct_link=True,
169        n_clusters=2,
170        cluster_encode=True,
171        type_clust="kmeans",
172        type_scaling=("std", "std", "std"),
173        lags=1,
174        type_pi="kde",
175        block_size=None,
176        replications=None,
177        kernel=None,
178        agg="mean",
179        seed=123,
180        backend="cpu",
181        show_progress=False,
182    ):
183        self.verbose = verbose
184        self.ignore_warnings = ignore_warnings
185        self.custom_metric = custom_metric
186        self.predictions = predictions
187        self.models = {}
188        self.random_state = random_state
189        self.estimators = estimators
190        self.preprocess = preprocess
191        self.n_layers = n_layers
192        self.h = h
193        super().__init__(
194            obj=obj,
195            n_hidden_features=n_hidden_features,
196            activation_name=activation_name,
197            a=a,
198            nodes_sim=nodes_sim,
199            bias=bias,
200            dropout=dropout,
201            direct_link=direct_link,
202            n_clusters=n_clusters,
203            cluster_encode=cluster_encode,
204            type_clust=type_clust,
205            type_scaling=type_scaling,
206            seed=seed,
207            backend=backend,
208            lags=lags,
209            type_pi=type_pi,
210            block_size=block_size,
211            replications=replications,
212            kernel=kernel,
213            agg=agg,
214            show_progress=show_progress,
215        )
216
217    def fit(self, X_train, X_test, xreg=None, per_series=False, **kwargs):
218        """Fit Regression algorithms to X_train, predict and score on X_test.
219
220        Parameters:
221
222            X_train: array-like or data frame,
223                Training vectors, where rows is the number of samples
224                and columns is the number of features.
225
226            X_test: array-like or data frame,
227                Testing vectors, where rows is the number of samples
228                and columns is the number of features.
229
230            xreg: array-like, optional (default=None)
231                Additional (external) regressors to be passed to self.obj
232                xreg must be in 'increasing' order (most recent observations last)
233
234            per_series: bool, optional (default=False)
235                When set to True, the metrics are computed series by series.
236
237            **kwargs: dict, optional (default=None)
238                Additional parameters to be passed to `fit` method of `obj`.
239
240        Returns:
241
242            scores: Pandas DataFrame
243                Returns metrics of all the models in a Pandas DataFrame.
244
245            predictions: Pandas DataFrame
246                Returns predictions of all the models in a Pandas DataFrame.
247
248        """
249        R2 = []
250        ADJR2 = []
251        ME = []
252        MPL = []
253        RMSE = []
254        MAE = []
255        MPE = []
256        MAPE = []
257        WINKLERSCORE = []
258        COVERAGE = []
259
260        # WIN = []
261        names = []
262        TIME = []
263        predictions = {}
264
265        if self.custom_metric:
266            CUSTOM_METRIC = []
267
268        if self.h is None:
269            assert X_test is not None, "If h is None, X_test must be provided."
270
271        if isinstance(X_train, np.ndarray):
272            X_train = pd.DataFrame(X_train)
273            X_test = pd.DataFrame(X_test)
274
275        self.series_names = X_train.columns.tolist()
276
277        X_train = convert_df_to_numeric(X_train)
278        X_test = convert_df_to_numeric(X_test)
279
280        numeric_features = X_train.select_dtypes(include=[np.number]).columns
281        categorical_features = X_train.select_dtypes(include=["object"]).columns
282
283        categorical_low, categorical_high = get_card_split(
284            X_train, categorical_features
285        )
286
287        if self.preprocess:
288            preprocessor = ColumnTransformer(
289                transformers=[
290                    ("numeric", numeric_transformer, numeric_features),
291                    (
292                        "categorical_low",
293                        categorical_transformer_low,
294                        categorical_low,
295                    ),
296                    (
297                        "categorical_high",
298                        categorical_transformer_high,
299                        categorical_high,
300                    ),
301                ]
302            )
303
304        if self.estimators == "all":
305            if self.n_layers <= 1:
306                self.regressors = REGRESSORSMTS
307            else:
308                self.regressors = DEEPREGRESSORSMTS
309        else:
310            if self.n_layers <= 1:
311                self.regressors = [
312                    ("MTS(" + est[0] + ")", est[1])
313                    for est in all_estimators()
314                    if (
315                        issubclass(est[1], RegressorMixin)
316                        and (est[0] in self.estimators)
317                    )
318                ]
319            else:  # self.n_layers > 1
320                self.regressors = [
321                    ("DeepMTS(" + est[0] + ")", est[1])
322                    for est in all_estimators()
323                    if (
324                        issubclass(est[1], RegressorMixin)
325                        and (est[0] in self.estimators)
326                    )
327                ]
328
329        if self.preprocess is True:
330            for name, model in tqdm(self.regressors):  # do parallel exec
331                start = time.time()
332                try:
333                    if "random_state" in model().get_params().keys():
334                        pipe = Pipeline(
335                            steps=[
336                                ("preprocessor", preprocessor),
337                                (
338                                    "regressor",
339                                    DeepMTS(
340                                        obj=model(
341                                            random_state=self.random_state,
342                                            **kwargs
343                                        ),
344                                        n_layers=self.n_layers,
345                                        n_hidden_features=self.n_hidden_features,
346                                        activation_name=self.activation_name,
347                                        a=self.a,
348                                        nodes_sim=self.nodes_sim,
349                                        bias=self.bias,
350                                        dropout=self.dropout,
351                                        direct_link=self.direct_link,
352                                        n_clusters=self.n_clusters,
353                                        cluster_encode=self.cluster_encode,
354                                        type_clust=self.type_clust,
355                                        type_scaling=self.type_scaling,
356                                        lags=self.lags,
357                                        type_pi=self.type_pi,
358                                        block_size=self.block_size,
359                                        replications=self.replications,
360                                        kernel=self.kernel,
361                                        agg=self.agg,
362                                        seed=self.seed,
363                                        backend=self.backend,
364                                        show_progress=self.show_progress,
365                                    ),
366                                ),
367                            ]
368                        )
369                    else:  # "random_state" in model().get_params().keys()
370                        pipe = Pipeline(
371                            steps=[
372                                ("preprocessor", preprocessor),
373                                (
374                                    "regressor",
375                                    DeepMTS(
376                                        obj=model(**kwargs),
377                                        n_layers=self.n_layers,
378                                        n_hidden_features=self.n_hidden_features,
379                                        activation_name=self.activation_name,
380                                        a=self.a,
381                                        nodes_sim=self.nodes_sim,
382                                        bias=self.bias,
383                                        dropout=self.dropout,
384                                        direct_link=self.direct_link,
385                                        n_clusters=self.n_clusters,
386                                        cluster_encode=self.cluster_encode,
387                                        type_clust=self.type_clust,
388                                        type_scaling=self.type_scaling,
389                                        lags=self.lags,
390                                        type_pi=self.type_pi,
391                                        block_size=self.block_size,
392                                        replications=self.replications,
393                                        kernel=self.kernel,
394                                        agg=self.agg,
395                                        seed=self.seed,
396                                        backend=self.backend,
397                                        show_progress=self.show_progress,
398                                    ),
399                                ),
400                            ]
401                        )
402
403                    pipe.fit(X_train, **kwargs)
404                    # pipe.fit(X_train, xreg=xreg)
405
406                    self.models[name] = pipe
407
408                    if self.h is None:
409                        X_pred = pipe["regressor"].predict(h=self.h, **kwargs)
410                    else:
411                        assert self.h > 0, "h must be > 0"
412                        X_pred = pipe["regressor"].predict(h=self.h, **kwargs)
413
414                    if (self.replications is not None) or (
415                        self.type_pi == "gaussian"
416                    ):
417                        if per_series == False:
418                            rmse = mean_squared_error(
419                                X_test, X_pred.mean, squared=False
420                            )
421                            mae = mean_absolute_error(X_test, X_pred.mean)
422                            mpl = mean_pinball_loss(X_test, X_pred.mean)
423                            winklerscore = winkler_score(
424                                obj=X_pred, actual=X_test, level=95
425                            )
426                            coveragecalc = coverage(X_pred, X_test, level=95)
427                        else:
428                            rmse = mean_errors(
429                                actual=X_test,
430                                pred=X_pred,
431                                scoring="root_mean_squared_error",
432                                per_series=True,
433                            )
434                            mae = mean_errors(
435                                actual=X_test,
436                                pred=X_pred,
437                                scoring="mean_absolute_error",
438                                per_series=True,
439                            )
440                            mpl = mean_errors(
441                                actual=X_test,
442                                pred=X_pred,
443                                scoring="mean_pinball_loss",
444                                per_series=True,
445                            )
446                            winklerscore = winkler_score(
447                                obj=X_pred,
448                                actual=X_test,
449                                level=95,
450                                per_series=True,
451                            )
452                            coveragecalc = coverage(
453                                X_pred, X_test, level=95, per_series=True
454                            )
455                    else:
456                        if per_series == False:
457                            rmse = mean_squared_error(
458                                X_test, X_pred, squared=False
459                            )
460                            mae = mean_absolute_error(X_test, X_pred)
461                            mpl = mean_pinball_loss(X_test, X_pred)
462                        else:
463                            rmse = mean_errors(
464                                actual=X_test,
465                                pred=X_pred,
466                                scoring="root_mean_squared_error",
467                                per_series=True,
468                            )
469                            mae = mean_errors(
470                                actual=X_test,
471                                pred=X_pred,
472                                scoring="mean_absolute_error",
473                                per_series=True,
474                            )
475                            mpl = mean_errors(
476                                actual=X_test,
477                                pred=X_pred,
478                                scoring="mean_pinball_loss",
479                                per_series=True,
480                            )
481
482                    names.append(name)
483                    RMSE.append(rmse)
484                    MAE.append(mae)
485                    MPL.append(mpl)
486
487                    if (self.replications is not None) or (
488                        self.type_pi == "gaussian"
489                    ):
490                        WINKLERSCORE.append(winklerscore)
491                        COVERAGE.append(coveragecalc)
492                    TIME.append(time.time() - start)
493
494                    if self.custom_metric:
495                        custom_metric = self.custom_metric(X_test, X_pred)
496                        CUSTOM_METRIC.append(custom_metric)
497
498                    if self.verbose > 0:
499                        if (self.replications is not None) or (
500                            self.type_pi == "gaussian"
501                        ):
502                            scores_verbose = {
503                                "Model": name,
504                                "RMSE": rmse,
505                                "MAE": mae,
506                                "MPL": mpl,
507                                "WINKLERSCORE": winklerscore,
508                                "COVERAGE": coveragecalc,
509                                "Time taken": time.time() - start,
510                            }
511                        else:
512                            scores_verbose = {
513                                "Model": name,
514                                "RMSE": rmse,
515                                "MAE": mae,
516                                "MPL": mpl,
517                                "Time taken": time.time() - start,
518                            }
519
520                        if self.custom_metric:
521                            scores_verbose[self.custom_metric.__name__] = (
522                                custom_metric
523                            )
524
525                        print(scores_verbose)
526                    if self.predictions:
527                        predictions[name] = X_pred
528                except Exception as exception:
529                    if self.ignore_warnings is False:
530                        print(name + " model failed to execute")
531                        print(exception)
532
533        else:  # no preprocessing
534
535            for name, model in tqdm(self.regressors):  # do parallel exec
536                start = time.time()
537                try:
538                    if "random_state" in model().get_params().keys():
539                        pipe = DeepMTS(
540                            obj=model(random_state=self.random_state, **kwargs),
541                            n_layers=self.n_layers,
542                            n_hidden_features=self.n_hidden_features,
543                            activation_name=self.activation_name,
544                            a=self.a,
545                            nodes_sim=self.nodes_sim,
546                            bias=self.bias,
547                            dropout=self.dropout,
548                            direct_link=self.direct_link,
549                            n_clusters=self.n_clusters,
550                            cluster_encode=self.cluster_encode,
551                            type_clust=self.type_clust,
552                            type_scaling=self.type_scaling,
553                            lags=self.lags,
554                            type_pi=self.type_pi,
555                            block_size=self.block_size,
556                            replications=self.replications,
557                            kernel=self.kernel,
558                            agg=self.agg,
559                            seed=self.seed,
560                            backend=self.backend,
561                            show_progress=self.show_progress,
562                        )
563                    else:
564                        pipe = DeepMTS(
565                            obj=model(**kwargs),
566                            n_layers=self.n_layers,
567                            n_hidden_features=self.n_hidden_features,
568                            activation_name=self.activation_name,
569                            a=self.a,
570                            nodes_sim=self.nodes_sim,
571                            bias=self.bias,
572                            dropout=self.dropout,
573                            direct_link=self.direct_link,
574                            n_clusters=self.n_clusters,
575                            cluster_encode=self.cluster_encode,
576                            type_clust=self.type_clust,
577                            type_scaling=self.type_scaling,
578                            lags=self.lags,
579                            type_pi=self.type_pi,
580                            block_size=self.block_size,
581                            replications=self.replications,
582                            kernel=self.kernel,
583                            agg=self.agg,
584                            seed=self.seed,
585                            backend=self.backend,
586                            show_progress=self.show_progress,
587                        )
588
589                    pipe.fit(X_train, xreg, **kwargs)
590                    # pipe.fit(X_train, xreg=xreg) # DO xreg like in `ahead`
591
592                    self.models[name] = pipe
593
594                    if self.preprocess is True:
595                        if self.h is None:
596                            X_pred = pipe["regressor"].predict(
597                                h=X_test.shape[0], **kwargs
598                            )
599                        else:
600                            assert (
601                                self.h > 0 and self.h < X_test.shape[0]
602                            ), "h must be > 0 and < X_test.shape[0]"
603                            X_pred = pipe["regressor"].predict(
604                                h=self.h, **kwargs
605                            )
606
607                    else:
608                        if self.h is None:
609                            X_pred = pipe.predict(
610                                h=X_test.shape[0], **kwargs
611                            )  # X_pred = pipe.predict(h=X_test.shape[0], new_xreg=new_xreg) ## DO xreg like in `ahead`
612                        else:
613                            assert (
614                                self.h > 0 and self.h < X_test.shape[0]
615                            ), "h must be > 0 and < X_test.shape[0]"
616                            X_pred = pipe.predict(h=self.h, **kwargs)
617
618                    if self.h is None:
619                        if (self.replications is not None) or (
620                            self.type_pi == "gaussian"
621                        ):
622
623                            if per_series == True:
624                                rmse = mean_errors(
625                                    actual=X_test,
626                                    pred=X_pred.mean,
627                                    scoring="root_mean_squared_error",
628                                    per_series=True,
629                                )
630                                mae = mean_errors(
631                                    actual=X_test,
632                                    pred=X_pred.mean,
633                                    scoring="mean_absolute_error",
634                                    per_series=True,
635                                )
636                                mpl = mean_errors(
637                                    actual=X_test,
638                                    pred=X_pred.mean,
639                                    scoring="mean_pinball_loss",
640                                    per_series=True,
641                                )
642                                winklerscore = winkler_score(
643                                    obj=X_pred,
644                                    actual=X_test,
645                                    level=95,
646                                    per_series=True,
647                                )
648                                coveragecalc = coverage(
649                                    X_pred, X_test, level=95, per_series=True
650                                )
651                            else:
652                                rmse = mean_squared_error(
653                                    X_test, X_pred.mean, squared=False
654                                )
655                                mae = mean_absolute_error(X_test, X_pred.mean)
656                                mpl = mean_pinball_loss(X_test, X_pred.mean)
657                                winklerscore = winkler_score(
658                                    obj=X_pred, actual=X_test, level=95
659                                )
660                                coveragecalc = coverage(
661                                    X_pred, X_test, level=95
662                                )
663                        else:  # no prediction interval
664                            if per_series == True:
665                                rmse = mean_errors(
666                                    actual=X_test,
667                                    pred=X_pred,
668                                    scoring="root_mean_squared_error",
669                                    per_series=True,
670                                )
671                                mae = mean_errors(
672                                    actual=X_test,
673                                    pred=X_pred,
674                                    scoring="mean_absolute_error",
675                                    per_series=True,
676                                )
677                                mpl = mean_errors(
678                                    actual=X_test,
679                                    pred=X_pred,
680                                    scoring="mean_pinball_loss",
681                                    per_series=True,
682                                )
683                            else:
684                                rmse = mean_squared_error(
685                                    X_test, X_pred, squared=False
686                                )
687                                mae = mean_absolute_error(X_test, X_pred)
688                                mpl = mean_pinball_loss(X_test, X_pred)
689                    else:  # self.h is not None
690                        if (self.replications is not None) or (
691                            self.type_pi == "gaussian"
692                        ):
693
694                            if per_series == False:
695                                if isinstance(X_test, pd.DataFrame) == False:
696                                    X_test_h = X_test[0: self.h, :]
697                                    rmse = mean_squared_error(
698                                        X_test_h, X_pred.mean, squared=False
699                                    )
700                                    mae = mean_absolute_error(
701                                        X_test_h, X_pred.mean
702                                    )
703                                    mpl = mean_pinball_loss(
704                                        X_test_h, X_pred.mean
705                                    )
706                                    winklerscore = winkler_score(
707                                        obj=X_pred, actual=X_test_h, level=95
708                                    )
709                                    coveragecalc = coverage(
710                                        X_pred, X_test_h, level=95
711                                    )
712                                else:
713                                    X_test_h = X_test.iloc[0: self.h, :]
714                                    rmse = mean_squared_error(
715                                        X_test_h, X_pred.mean, squared=False
716                                    )
717                                    mae = mean_absolute_error(
718                                        X_test_h, X_pred.mean
719                                    )
720                                    mpl = mean_pinball_loss(
721                                        X_test_h, X_pred.mean
722                                    )
723                                    winklerscore = winkler_score(
724                                        obj=X_pred, actual=X_test_h, level=95
725                                    )
726                                    coveragecalc = coverage(
727                                        X_pred, X_test_h, level=95
728                                    )
729                            else:
730                                if isinstance(X_test, pd.DataFrame):
731                                    X_test_h = X_test.iloc[0: self.h, :]
732                                    rmse = mean_errors(
733                                        actual=X_test_h,
734                                        pred=X_pred,
735                                        scoring="root_mean_squared_error",
736                                        per_series=True,
737                                    )
738                                    mae = mean_errors(
739                                        actual=X_test_h,
740                                        pred=X_pred,
741                                        scoring="mean_absolute_error",
742                                        per_series=True,
743                                    )
744                                    mpl = mean_errors(
745                                        actual=X_test_h,
746                                        pred=X_pred,
747                                        scoring="mean_pinball_loss",
748                                        per_series=True,
749                                    )
750                                    winklerscore = winkler_score(
751                                        obj=X_pred,
752                                        actual=X_test_h,
753                                        level=95,
754                                        per_series=True,
755                                    )
756                                    coveragecalc = coverage(
757                                        X_pred,
758                                        X_test_h,
759                                        level=95,
760                                        per_series=True,
761                                    )
762                                else:
763                                    X_test_h = X_test[0: self.h, :]
764                                    rmse = mean_errors(
765                                        actual=X_test_h,
766                                        pred=X_pred,
767                                        scoring="root_mean_squared_error",
768                                        per_series=True,
769                                    )
770                                    mae = mean_errors(
771                                        actual=X_test_h,
772                                        pred=X_pred,
773                                        scoring="mean_absolute_error",
774                                        per_series=True,
775                                    )
776                                    mpl = mean_errors(
777                                        actual=X_test_h,
778                                        pred=X_pred,
779                                        scoring="mean_pinball_loss",
780                                        per_series=True,
781                                    )
782                                    winklerscore = winkler_score(
783                                        obj=X_pred,
784                                        actual=X_test_h,
785                                        level=95,
786                                        per_series=True,
787                                    )
788                                    coveragecalc = coverage(
789                                        X_pred,
790                                        X_test_h,
791                                        level=95,
792                                        per_series=True,
793                                    )
794                        else:  # no prediction interval
795                            if per_series == False:
796                                if isinstance(X_test, pd.DataFrame):
797                                    X_test_h = X_test.iloc[0: self.h, :]
798                                    rmse = mean_squared_error(
799                                        X_test_h, X_pred, squared=False
800                                    )
801                                    mae = mean_absolute_error(X_test_h, X_pred)
802                                    mpl = mean_pinball_loss(X_test_h, X_pred)
803                                else:
804                                    X_test_h = X_test[0: self.h, :]
805                                    rmse = mean_squared_error(
806                                        X_test_h, X_pred, squared=False
807                                    )
808                                    mae = mean_absolute_error(X_test_h, X_pred)
809                                    mpl = mean_pinball_loss(X_test_h, X_pred)
810                            else:
811                                if isinstance(X_test, pd.DataFrame):
812                                    X_test_h = X_test.iloc[0: self.h, :]
813                                    rmse = mean_errors(
814                                        actual=X_test_h,
815                                        pred=X_pred,
816                                        scoring="root_mean_squared_error",
817                                        per_series=True,
818                                    )
819                                    mae = mean_errors(
820                                        actual=X_test_h,
821                                        pred=X_pred,
822                                        scoring="mean_absolute_error",
823                                        per_series=True,
824                                    )
825                                    mpl = mean_errors(
826                                        actual=X_test_h,
827                                        pred=X_pred,
828                                        scoring="mean_pinball_loss",
829                                        per_series=True,
830                                    )
831                                else:
832                                    X_test_h = X_test[0: self.h, :]
833                                    rmse = mean_errors(
834                                        actual=X_test_h,
835                                        pred=X_pred,
836                                        scoring="root_mean_squared_error",
837                                        per_series=True,
838                                    )
839                                    mae = mean_errors(
840                                        actual=X_test_h,
841                                        pred=X_pred,
842                                        scoring="mean_absolute_error",
843                                        per_series=True,
844                                    )
845
846                    names.append(name)
847                    RMSE.append(rmse)
848                    MAE.append(mae)
849                    MPL.append(mpl)
850                    if (self.replications is not None) or (
851                        self.type_pi == "gaussian"
852                    ):
853                        WINKLERSCORE.append(winklerscore)
854                        COVERAGE.append(coveragecalc)
855                    TIME.append(time.time() - start)
856
857                    if self.custom_metric:
858                        if self.h is None:
859                            custom_metric = self.custom_metric(X_test, X_pred)
860                        else:
861                            custom_metric = self.custom_metric(X_test_h, X_pred)
862
863                        CUSTOM_METRIC.append(custom_metric)
864
865                    if self.verbose > 0:
866                        if (self.replications is not None) or (
867                            self.type_pi == "gaussian"
868                        ):
869                            scores_verbose = {
870                                "Model": name,
871                                "RMSE": rmse,
872                                "MAE": mae,
873                                "MPL": mpl,
874                                "WINKLERSCORE": winklerscore,
875                                "COVERAGE": coveragecalc,
876                                "Time taken": time.time() - start,
877                            }
878                        else:
879                            scores_verbose = {
880                                "Model": name,
881                                "RMSE": rmse,
882                                "MAE": mae,
883                                "MPL": mpl,
884                                "Time taken": time.time() - start,
885                            }
886
887                        if self.custom_metric:
888                            scores_verbose[self.custom_metric.__name__] = (
889                                custom_metric
890                            )
891
892                        print(scores_verbose)
893
894                    if self.predictions:
895                        predictions[name] = X_pred
896
897                except Exception as exception:
898                    if self.ignore_warnings is False:
899                        print(name + " model failed to execute")
900                        print(exception)
901
902        if (self.replications is not None) or (self.type_pi == "gaussian"):
903            scores = {
904                "Model": names,
905                "RMSE": RMSE,
906                "MAE": MAE,
907                "MPL": MPL,
908                "WINKLERSCORE": WINKLERSCORE,
909                "COVERAGE": COVERAGE,
910                "Time Taken": TIME,
911            }
912        else:
913            scores = {
914                "Model": names,
915                "RMSE": RMSE,
916                "MAE": MAE,
917                "MPL": MPL,
918                "Time Taken": TIME,
919            }
920
921        if self.custom_metric:
922            scores[self.custom_metric.__name__] = CUSTOM_METRIC
923
924        if per_series:
925            scores = dict_to_dataframe_series(scores, self.series_names)
926        else:
927            scores = pd.DataFrame(scores)
928
929        try:
930            scores = scores.sort_values(
931                by=self.sort_by, ascending=True
932            ).set_index("Model")
933        except:
934            pass
935
936        if self.predictions:
937            predictions_df = pd.DataFrame.from_dict(predictions)
938
939        return scores, predictions_df if self.predictions is True else scores
940
941    def provide_models(self, X_train, X_test):
942        """
943        This function returns all the model objects trained in fit function.
944        If fit is not called already, then we call fit and then return the models.
945
946        Parameters:
947
948            X_train : array-like,
949                Training vectors, where rows is the number of samples
950                and columns is the number of features.
951
952            X_test : array-like,
953                Testing vectors, where rows is the number of samples
954                and columns is the number of features.
955
956        Returns:
957
958            models: dict-object,
959                Returns a dictionary with each model pipeline as value
960                with key as name of models.
961
962        """
963        if self.h is None:
964            if len(self.models.keys()) == 0:
965                self.fit(X_train, X_test)
966        else:
967            if len(self.models.keys()) == 0:
968                if isinstance(X_test, pd.DataFrame):
969                    self.fit(X_train, X_test.iloc[0: self.h, :])
970                else:
971                    self.fit(X_train, X_test[0: self.h, :])
972
973        return self.models

Fitting -- almost -- all the regression algorithms with layers of nnetsauce's CustomRegressor to multivariate time series and returning their scores.

Parameters:

verbose: int, optional (default=0)
    Any positive number for verbosity.

ignore_warnings: bool, optional (default=True)
    When set to True, the warning related to algorigms that are not
    able to run are ignored.

custom_metric: function, optional (default=None)
    When function is provided, models are evaluated based on the custom
      evaluation metric provided.

predictions: bool, optional (default=False)
    When set to True, the predictions of all the models models are returned as dataframe.

sort_by: string, optional (default='RMSE')
    Sort models by a metric. Available options are 'RMSE', 'MAE', 'MPL', 'MPE', 'MAPE',
    'R-Squared', 'Adjusted R-Squared' or a custom metric identified by its name and
    provided by custom_metric.

random_state: int, optional (default=42)
    Reproducibiility seed.

estimators: list, optional (default='all')
    list of Estimators (regression algorithms) names or just 'all' (default='all')

preprocess: bool, preprocessing is done when set to True

n_layers: int, optional (default=1)
    Number of layers in the network. When set to 1, the model is equivalent to a MTS.

h: int, optional (default=None)
    Number of steps ahead to predict (when used, must be > 0 and < X_test.shape[0]).

All the other parameters are the same as MTS's.

Examples:

See https://thierrymoudiki.github.io/blog/2023/10/29/python/quasirandomizednn/MTS-LazyPredict
def fit(self, X_train, X_test, xreg=None, per_series=False, **kwargs):
217    def fit(self, X_train, X_test, xreg=None, per_series=False, **kwargs):
218        """Fit Regression algorithms to X_train, predict and score on X_test.
219
220        Parameters:
221
222            X_train: array-like or data frame,
223                Training vectors, where rows is the number of samples
224                and columns is the number of features.
225
226            X_test: array-like or data frame,
227                Testing vectors, where rows is the number of samples
228                and columns is the number of features.
229
230            xreg: array-like, optional (default=None)
231                Additional (external) regressors to be passed to self.obj
232                xreg must be in 'increasing' order (most recent observations last)
233
234            per_series: bool, optional (default=False)
235                When set to True, the metrics are computed series by series.
236
237            **kwargs: dict, optional (default=None)
238                Additional parameters to be passed to `fit` method of `obj`.
239
240        Returns:
241
242            scores: Pandas DataFrame
243                Returns metrics of all the models in a Pandas DataFrame.
244
245            predictions: Pandas DataFrame
246                Returns predictions of all the models in a Pandas DataFrame.
247
248        """
249        R2 = []
250        ADJR2 = []
251        ME = []
252        MPL = []
253        RMSE = []
254        MAE = []
255        MPE = []
256        MAPE = []
257        WINKLERSCORE = []
258        COVERAGE = []
259
260        # WIN = []
261        names = []
262        TIME = []
263        predictions = {}
264
265        if self.custom_metric:
266            CUSTOM_METRIC = []
267
268        if self.h is None:
269            assert X_test is not None, "If h is None, X_test must be provided."
270
271        if isinstance(X_train, np.ndarray):
272            X_train = pd.DataFrame(X_train)
273            X_test = pd.DataFrame(X_test)
274
275        self.series_names = X_train.columns.tolist()
276
277        X_train = convert_df_to_numeric(X_train)
278        X_test = convert_df_to_numeric(X_test)
279
280        numeric_features = X_train.select_dtypes(include=[np.number]).columns
281        categorical_features = X_train.select_dtypes(include=["object"]).columns
282
283        categorical_low, categorical_high = get_card_split(
284            X_train, categorical_features
285        )
286
287        if self.preprocess:
288            preprocessor = ColumnTransformer(
289                transformers=[
290                    ("numeric", numeric_transformer, numeric_features),
291                    (
292                        "categorical_low",
293                        categorical_transformer_low,
294                        categorical_low,
295                    ),
296                    (
297                        "categorical_high",
298                        categorical_transformer_high,
299                        categorical_high,
300                    ),
301                ]
302            )
303
304        if self.estimators == "all":
305            if self.n_layers <= 1:
306                self.regressors = REGRESSORSMTS
307            else:
308                self.regressors = DEEPREGRESSORSMTS
309        else:
310            if self.n_layers <= 1:
311                self.regressors = [
312                    ("MTS(" + est[0] + ")", est[1])
313                    for est in all_estimators()
314                    if (
315                        issubclass(est[1], RegressorMixin)
316                        and (est[0] in self.estimators)
317                    )
318                ]
319            else:  # self.n_layers > 1
320                self.regressors = [
321                    ("DeepMTS(" + est[0] + ")", est[1])
322                    for est in all_estimators()
323                    if (
324                        issubclass(est[1], RegressorMixin)
325                        and (est[0] in self.estimators)
326                    )
327                ]
328
329        if self.preprocess is True:
330            for name, model in tqdm(self.regressors):  # do parallel exec
331                start = time.time()
332                try:
333                    if "random_state" in model().get_params().keys():
334                        pipe = Pipeline(
335                            steps=[
336                                ("preprocessor", preprocessor),
337                                (
338                                    "regressor",
339                                    DeepMTS(
340                                        obj=model(
341                                            random_state=self.random_state,
342                                            **kwargs
343                                        ),
344                                        n_layers=self.n_layers,
345                                        n_hidden_features=self.n_hidden_features,
346                                        activation_name=self.activation_name,
347                                        a=self.a,
348                                        nodes_sim=self.nodes_sim,
349                                        bias=self.bias,
350                                        dropout=self.dropout,
351                                        direct_link=self.direct_link,
352                                        n_clusters=self.n_clusters,
353                                        cluster_encode=self.cluster_encode,
354                                        type_clust=self.type_clust,
355                                        type_scaling=self.type_scaling,
356                                        lags=self.lags,
357                                        type_pi=self.type_pi,
358                                        block_size=self.block_size,
359                                        replications=self.replications,
360                                        kernel=self.kernel,
361                                        agg=self.agg,
362                                        seed=self.seed,
363                                        backend=self.backend,
364                                        show_progress=self.show_progress,
365                                    ),
366                                ),
367                            ]
368                        )
369                    else:  # "random_state" in model().get_params().keys()
370                        pipe = Pipeline(
371                            steps=[
372                                ("preprocessor", preprocessor),
373                                (
374                                    "regressor",
375                                    DeepMTS(
376                                        obj=model(**kwargs),
377                                        n_layers=self.n_layers,
378                                        n_hidden_features=self.n_hidden_features,
379                                        activation_name=self.activation_name,
380                                        a=self.a,
381                                        nodes_sim=self.nodes_sim,
382                                        bias=self.bias,
383                                        dropout=self.dropout,
384                                        direct_link=self.direct_link,
385                                        n_clusters=self.n_clusters,
386                                        cluster_encode=self.cluster_encode,
387                                        type_clust=self.type_clust,
388                                        type_scaling=self.type_scaling,
389                                        lags=self.lags,
390                                        type_pi=self.type_pi,
391                                        block_size=self.block_size,
392                                        replications=self.replications,
393                                        kernel=self.kernel,
394                                        agg=self.agg,
395                                        seed=self.seed,
396                                        backend=self.backend,
397                                        show_progress=self.show_progress,
398                                    ),
399                                ),
400                            ]
401                        )
402
403                    pipe.fit(X_train, **kwargs)
404                    # pipe.fit(X_train, xreg=xreg)
405
406                    self.models[name] = pipe
407
408                    if self.h is None:
409                        X_pred = pipe["regressor"].predict(h=self.h, **kwargs)
410                    else:
411                        assert self.h > 0, "h must be > 0"
412                        X_pred = pipe["regressor"].predict(h=self.h, **kwargs)
413
414                    if (self.replications is not None) or (
415                        self.type_pi == "gaussian"
416                    ):
417                        if per_series == False:
418                            rmse = mean_squared_error(
419                                X_test, X_pred.mean, squared=False
420                            )
421                            mae = mean_absolute_error(X_test, X_pred.mean)
422                            mpl = mean_pinball_loss(X_test, X_pred.mean)
423                            winklerscore = winkler_score(
424                                obj=X_pred, actual=X_test, level=95
425                            )
426                            coveragecalc = coverage(X_pred, X_test, level=95)
427                        else:
428                            rmse = mean_errors(
429                                actual=X_test,
430                                pred=X_pred,
431                                scoring="root_mean_squared_error",
432                                per_series=True,
433                            )
434                            mae = mean_errors(
435                                actual=X_test,
436                                pred=X_pred,
437                                scoring="mean_absolute_error",
438                                per_series=True,
439                            )
440                            mpl = mean_errors(
441                                actual=X_test,
442                                pred=X_pred,
443                                scoring="mean_pinball_loss",
444                                per_series=True,
445                            )
446                            winklerscore = winkler_score(
447                                obj=X_pred,
448                                actual=X_test,
449                                level=95,
450                                per_series=True,
451                            )
452                            coveragecalc = coverage(
453                                X_pred, X_test, level=95, per_series=True
454                            )
455                    else:
456                        if per_series == False:
457                            rmse = mean_squared_error(
458                                X_test, X_pred, squared=False
459                            )
460                            mae = mean_absolute_error(X_test, X_pred)
461                            mpl = mean_pinball_loss(X_test, X_pred)
462                        else:
463                            rmse = mean_errors(
464                                actual=X_test,
465                                pred=X_pred,
466                                scoring="root_mean_squared_error",
467                                per_series=True,
468                            )
469                            mae = mean_errors(
470                                actual=X_test,
471                                pred=X_pred,
472                                scoring="mean_absolute_error",
473                                per_series=True,
474                            )
475                            mpl = mean_errors(
476                                actual=X_test,
477                                pred=X_pred,
478                                scoring="mean_pinball_loss",
479                                per_series=True,
480                            )
481
482                    names.append(name)
483                    RMSE.append(rmse)
484                    MAE.append(mae)
485                    MPL.append(mpl)
486
487                    if (self.replications is not None) or (
488                        self.type_pi == "gaussian"
489                    ):
490                        WINKLERSCORE.append(winklerscore)
491                        COVERAGE.append(coveragecalc)
492                    TIME.append(time.time() - start)
493
494                    if self.custom_metric:
495                        custom_metric = self.custom_metric(X_test, X_pred)
496                        CUSTOM_METRIC.append(custom_metric)
497
498                    if self.verbose > 0:
499                        if (self.replications is not None) or (
500                            self.type_pi == "gaussian"
501                        ):
502                            scores_verbose = {
503                                "Model": name,
504                                "RMSE": rmse,
505                                "MAE": mae,
506                                "MPL": mpl,
507                                "WINKLERSCORE": winklerscore,
508                                "COVERAGE": coveragecalc,
509                                "Time taken": time.time() - start,
510                            }
511                        else:
512                            scores_verbose = {
513                                "Model": name,
514                                "RMSE": rmse,
515                                "MAE": mae,
516                                "MPL": mpl,
517                                "Time taken": time.time() - start,
518                            }
519
520                        if self.custom_metric:
521                            scores_verbose[self.custom_metric.__name__] = (
522                                custom_metric
523                            )
524
525                        print(scores_verbose)
526                    if self.predictions:
527                        predictions[name] = X_pred
528                except Exception as exception:
529                    if self.ignore_warnings is False:
530                        print(name + " model failed to execute")
531                        print(exception)
532
533        else:  # no preprocessing
534
535            for name, model in tqdm(self.regressors):  # do parallel exec
536                start = time.time()
537                try:
538                    if "random_state" in model().get_params().keys():
539                        pipe = DeepMTS(
540                            obj=model(random_state=self.random_state, **kwargs),
541                            n_layers=self.n_layers,
542                            n_hidden_features=self.n_hidden_features,
543                            activation_name=self.activation_name,
544                            a=self.a,
545                            nodes_sim=self.nodes_sim,
546                            bias=self.bias,
547                            dropout=self.dropout,
548                            direct_link=self.direct_link,
549                            n_clusters=self.n_clusters,
550                            cluster_encode=self.cluster_encode,
551                            type_clust=self.type_clust,
552                            type_scaling=self.type_scaling,
553                            lags=self.lags,
554                            type_pi=self.type_pi,
555                            block_size=self.block_size,
556                            replications=self.replications,
557                            kernel=self.kernel,
558                            agg=self.agg,
559                            seed=self.seed,
560                            backend=self.backend,
561                            show_progress=self.show_progress,
562                        )
563                    else:
564                        pipe = DeepMTS(
565                            obj=model(**kwargs),
566                            n_layers=self.n_layers,
567                            n_hidden_features=self.n_hidden_features,
568                            activation_name=self.activation_name,
569                            a=self.a,
570                            nodes_sim=self.nodes_sim,
571                            bias=self.bias,
572                            dropout=self.dropout,
573                            direct_link=self.direct_link,
574                            n_clusters=self.n_clusters,
575                            cluster_encode=self.cluster_encode,
576                            type_clust=self.type_clust,
577                            type_scaling=self.type_scaling,
578                            lags=self.lags,
579                            type_pi=self.type_pi,
580                            block_size=self.block_size,
581                            replications=self.replications,
582                            kernel=self.kernel,
583                            agg=self.agg,
584                            seed=self.seed,
585                            backend=self.backend,
586                            show_progress=self.show_progress,
587                        )
588
589                    pipe.fit(X_train, xreg, **kwargs)
590                    # pipe.fit(X_train, xreg=xreg) # DO xreg like in `ahead`
591
592                    self.models[name] = pipe
593
594                    if self.preprocess is True:
595                        if self.h is None:
596                            X_pred = pipe["regressor"].predict(
597                                h=X_test.shape[0], **kwargs
598                            )
599                        else:
600                            assert (
601                                self.h > 0 and self.h < X_test.shape[0]
602                            ), "h must be > 0 and < X_test.shape[0]"
603                            X_pred = pipe["regressor"].predict(
604                                h=self.h, **kwargs
605                            )
606
607                    else:
608                        if self.h is None:
609                            X_pred = pipe.predict(
610                                h=X_test.shape[0], **kwargs
611                            )  # X_pred = pipe.predict(h=X_test.shape[0], new_xreg=new_xreg) ## DO xreg like in `ahead`
612                        else:
613                            assert (
614                                self.h > 0 and self.h < X_test.shape[0]
615                            ), "h must be > 0 and < X_test.shape[0]"
616                            X_pred = pipe.predict(h=self.h, **kwargs)
617
618                    if self.h is None:
619                        if (self.replications is not None) or (
620                            self.type_pi == "gaussian"
621                        ):
622
623                            if per_series == True:
624                                rmse = mean_errors(
625                                    actual=X_test,
626                                    pred=X_pred.mean,
627                                    scoring="root_mean_squared_error",
628                                    per_series=True,
629                                )
630                                mae = mean_errors(
631                                    actual=X_test,
632                                    pred=X_pred.mean,
633                                    scoring="mean_absolute_error",
634                                    per_series=True,
635                                )
636                                mpl = mean_errors(
637                                    actual=X_test,
638                                    pred=X_pred.mean,
639                                    scoring="mean_pinball_loss",
640                                    per_series=True,
641                                )
642                                winklerscore = winkler_score(
643                                    obj=X_pred,
644                                    actual=X_test,
645                                    level=95,
646                                    per_series=True,
647                                )
648                                coveragecalc = coverage(
649                                    X_pred, X_test, level=95, per_series=True
650                                )
651                            else:
652                                rmse = mean_squared_error(
653                                    X_test, X_pred.mean, squared=False
654                                )
655                                mae = mean_absolute_error(X_test, X_pred.mean)
656                                mpl = mean_pinball_loss(X_test, X_pred.mean)
657                                winklerscore = winkler_score(
658                                    obj=X_pred, actual=X_test, level=95
659                                )
660                                coveragecalc = coverage(
661                                    X_pred, X_test, level=95
662                                )
663                        else:  # no prediction interval
664                            if per_series == True:
665                                rmse = mean_errors(
666                                    actual=X_test,
667                                    pred=X_pred,
668                                    scoring="root_mean_squared_error",
669                                    per_series=True,
670                                )
671                                mae = mean_errors(
672                                    actual=X_test,
673                                    pred=X_pred,
674                                    scoring="mean_absolute_error",
675                                    per_series=True,
676                                )
677                                mpl = mean_errors(
678                                    actual=X_test,
679                                    pred=X_pred,
680                                    scoring="mean_pinball_loss",
681                                    per_series=True,
682                                )
683                            else:
684                                rmse = mean_squared_error(
685                                    X_test, X_pred, squared=False
686                                )
687                                mae = mean_absolute_error(X_test, X_pred)
688                                mpl = mean_pinball_loss(X_test, X_pred)
689                    else:  # self.h is not None
690                        if (self.replications is not None) or (
691                            self.type_pi == "gaussian"
692                        ):
693
694                            if per_series == False:
695                                if isinstance(X_test, pd.DataFrame) == False:
696                                    X_test_h = X_test[0: self.h, :]
697                                    rmse = mean_squared_error(
698                                        X_test_h, X_pred.mean, squared=False
699                                    )
700                                    mae = mean_absolute_error(
701                                        X_test_h, X_pred.mean
702                                    )
703                                    mpl = mean_pinball_loss(
704                                        X_test_h, X_pred.mean
705                                    )
706                                    winklerscore = winkler_score(
707                                        obj=X_pred, actual=X_test_h, level=95
708                                    )
709                                    coveragecalc = coverage(
710                                        X_pred, X_test_h, level=95
711                                    )
712                                else:
713                                    X_test_h = X_test.iloc[0: self.h, :]
714                                    rmse = mean_squared_error(
715                                        X_test_h, X_pred.mean, squared=False
716                                    )
717                                    mae = mean_absolute_error(
718                                        X_test_h, X_pred.mean
719                                    )
720                                    mpl = mean_pinball_loss(
721                                        X_test_h, X_pred.mean
722                                    )
723                                    winklerscore = winkler_score(
724                                        obj=X_pred, actual=X_test_h, level=95
725                                    )
726                                    coveragecalc = coverage(
727                                        X_pred, X_test_h, level=95
728                                    )
729                            else:
730                                if isinstance(X_test, pd.DataFrame):
731                                    X_test_h = X_test.iloc[0: self.h, :]
732                                    rmse = mean_errors(
733                                        actual=X_test_h,
734                                        pred=X_pred,
735                                        scoring="root_mean_squared_error",
736                                        per_series=True,
737                                    )
738                                    mae = mean_errors(
739                                        actual=X_test_h,
740                                        pred=X_pred,
741                                        scoring="mean_absolute_error",
742                                        per_series=True,
743                                    )
744                                    mpl = mean_errors(
745                                        actual=X_test_h,
746                                        pred=X_pred,
747                                        scoring="mean_pinball_loss",
748                                        per_series=True,
749                                    )
750                                    winklerscore = winkler_score(
751                                        obj=X_pred,
752                                        actual=X_test_h,
753                                        level=95,
754                                        per_series=True,
755                                    )
756                                    coveragecalc = coverage(
757                                        X_pred,
758                                        X_test_h,
759                                        level=95,
760                                        per_series=True,
761                                    )
762                                else:
763                                    X_test_h = X_test[0: self.h, :]
764                                    rmse = mean_errors(
765                                        actual=X_test_h,
766                                        pred=X_pred,
767                                        scoring="root_mean_squared_error",
768                                        per_series=True,
769                                    )
770                                    mae = mean_errors(
771                                        actual=X_test_h,
772                                        pred=X_pred,
773                                        scoring="mean_absolute_error",
774                                        per_series=True,
775                                    )
776                                    mpl = mean_errors(
777                                        actual=X_test_h,
778                                        pred=X_pred,
779                                        scoring="mean_pinball_loss",
780                                        per_series=True,
781                                    )
782                                    winklerscore = winkler_score(
783                                        obj=X_pred,
784                                        actual=X_test_h,
785                                        level=95,
786                                        per_series=True,
787                                    )
788                                    coveragecalc = coverage(
789                                        X_pred,
790                                        X_test_h,
791                                        level=95,
792                                        per_series=True,
793                                    )
794                        else:  # no prediction interval
795                            if per_series == False:
796                                if isinstance(X_test, pd.DataFrame):
797                                    X_test_h = X_test.iloc[0: self.h, :]
798                                    rmse = mean_squared_error(
799                                        X_test_h, X_pred, squared=False
800                                    )
801                                    mae = mean_absolute_error(X_test_h, X_pred)
802                                    mpl = mean_pinball_loss(X_test_h, X_pred)
803                                else:
804                                    X_test_h = X_test[0: self.h, :]
805                                    rmse = mean_squared_error(
806                                        X_test_h, X_pred, squared=False
807                                    )
808                                    mae = mean_absolute_error(X_test_h, X_pred)
809                                    mpl = mean_pinball_loss(X_test_h, X_pred)
810                            else:
811                                if isinstance(X_test, pd.DataFrame):
812                                    X_test_h = X_test.iloc[0: self.h, :]
813                                    rmse = mean_errors(
814                                        actual=X_test_h,
815                                        pred=X_pred,
816                                        scoring="root_mean_squared_error",
817                                        per_series=True,
818                                    )
819                                    mae = mean_errors(
820                                        actual=X_test_h,
821                                        pred=X_pred,
822                                        scoring="mean_absolute_error",
823                                        per_series=True,
824                                    )
825                                    mpl = mean_errors(
826                                        actual=X_test_h,
827                                        pred=X_pred,
828                                        scoring="mean_pinball_loss",
829                                        per_series=True,
830                                    )
831                                else:
832                                    X_test_h = X_test[0: self.h, :]
833                                    rmse = mean_errors(
834                                        actual=X_test_h,
835                                        pred=X_pred,
836                                        scoring="root_mean_squared_error",
837                                        per_series=True,
838                                    )
839                                    mae = mean_errors(
840                                        actual=X_test_h,
841                                        pred=X_pred,
842                                        scoring="mean_absolute_error",
843                                        per_series=True,
844                                    )
845
846                    names.append(name)
847                    RMSE.append(rmse)
848                    MAE.append(mae)
849                    MPL.append(mpl)
850                    if (self.replications is not None) or (
851                        self.type_pi == "gaussian"
852                    ):
853                        WINKLERSCORE.append(winklerscore)
854                        COVERAGE.append(coveragecalc)
855                    TIME.append(time.time() - start)
856
857                    if self.custom_metric:
858                        if self.h is None:
859                            custom_metric = self.custom_metric(X_test, X_pred)
860                        else:
861                            custom_metric = self.custom_metric(X_test_h, X_pred)
862
863                        CUSTOM_METRIC.append(custom_metric)
864
865                    if self.verbose > 0:
866                        if (self.replications is not None) or (
867                            self.type_pi == "gaussian"
868                        ):
869                            scores_verbose = {
870                                "Model": name,
871                                "RMSE": rmse,
872                                "MAE": mae,
873                                "MPL": mpl,
874                                "WINKLERSCORE": winklerscore,
875                                "COVERAGE": coveragecalc,
876                                "Time taken": time.time() - start,
877                            }
878                        else:
879                            scores_verbose = {
880                                "Model": name,
881                                "RMSE": rmse,
882                                "MAE": mae,
883                                "MPL": mpl,
884                                "Time taken": time.time() - start,
885                            }
886
887                        if self.custom_metric:
888                            scores_verbose[self.custom_metric.__name__] = (
889                                custom_metric
890                            )
891
892                        print(scores_verbose)
893
894                    if self.predictions:
895                        predictions[name] = X_pred
896
897                except Exception as exception:
898                    if self.ignore_warnings is False:
899                        print(name + " model failed to execute")
900                        print(exception)
901
902        if (self.replications is not None) or (self.type_pi == "gaussian"):
903            scores = {
904                "Model": names,
905                "RMSE": RMSE,
906                "MAE": MAE,
907                "MPL": MPL,
908                "WINKLERSCORE": WINKLERSCORE,
909                "COVERAGE": COVERAGE,
910                "Time Taken": TIME,
911            }
912        else:
913            scores = {
914                "Model": names,
915                "RMSE": RMSE,
916                "MAE": MAE,
917                "MPL": MPL,
918                "Time Taken": TIME,
919            }
920
921        if self.custom_metric:
922            scores[self.custom_metric.__name__] = CUSTOM_METRIC
923
924        if per_series:
925            scores = dict_to_dataframe_series(scores, self.series_names)
926        else:
927            scores = pd.DataFrame(scores)
928
929        try:
930            scores = scores.sort_values(
931                by=self.sort_by, ascending=True
932            ).set_index("Model")
933        except:
934            pass
935
936        if self.predictions:
937            predictions_df = pd.DataFrame.from_dict(predictions)
938
939        return scores, predictions_df if self.predictions is True else scores

Fit Regression algorithms to X_train, predict and score on X_test.

Parameters:

X_train: array-like or data frame,
    Training vectors, where rows is the number of samples
    and columns is the number of features.

X_test: array-like or data frame,
    Testing vectors, where rows is the number of samples
    and columns is the number of features.

xreg: array-like, optional (default=None)
    Additional (external) regressors to be passed to self.obj
    xreg must be in 'increasing' order (most recent observations last)

per_series: bool, optional (default=False)
    When set to True, the metrics are computed series by series.

**kwargs: dict, optional (default=None)
    Additional parameters to be passed to `fit` method of `obj`.

Returns:

scores: Pandas DataFrame
    Returns metrics of all the models in a Pandas DataFrame.

predictions: Pandas DataFrame
    Returns predictions of all the models in a Pandas DataFrame.
def provide_models(self, X_train, X_test):
941    def provide_models(self, X_train, X_test):
942        """
943        This function returns all the model objects trained in fit function.
944        If fit is not called already, then we call fit and then return the models.
945
946        Parameters:
947
948            X_train : array-like,
949                Training vectors, where rows is the number of samples
950                and columns is the number of features.
951
952            X_test : array-like,
953                Testing vectors, where rows is the number of samples
954                and columns is the number of features.
955
956        Returns:
957
958            models: dict-object,
959                Returns a dictionary with each model pipeline as value
960                with key as name of models.
961
962        """
963        if self.h is None:
964            if len(self.models.keys()) == 0:
965                self.fit(X_train, X_test)
966        else:
967            if len(self.models.keys()) == 0:
968                if isinstance(X_test, pd.DataFrame):
969                    self.fit(X_train, X_test.iloc[0: self.h, :])
970                else:
971                    self.fit(X_train, X_test[0: self.h, :])
972
973        return self.models

This function returns all the model objects trained in fit function. If fit is not called already, then we call fit and then return the models.

Parameters:

X_train : array-like,
    Training vectors, where rows is the number of samples
    and columns is the number of features.

X_test : array-like,
    Testing vectors, where rows is the number of samples
    and columns is the number of features.

Returns:

models: dict-object,
    Returns a dictionary with each model pipeline as value
    with key as name of models.
class MTS(nnetsauce.Base):
  33class MTS(Base):
  34    """Univariate and multivariate time series (MTS) forecasting with Quasi-Randomized networks
  35
  36    Parameters:
  37
  38        obj: object.
  39            any object containing a method fit (obj.fit()) and a method predict
  40            (obj.predict()).
  41
  42        n_hidden_features: int.
  43            number of nodes in the hidden layer.
  44
  45        activation_name: str.
  46            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'.
  47
  48        a: float.
  49            hyperparameter for 'prelu' or 'elu' activation function.
  50
  51        nodes_sim: str.
  52            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
  53            'uniform'.
  54
  55        bias: boolean.
  56            indicates if the hidden layer contains a bias term (True) or not
  57            (False).
  58
  59        dropout: float.
  60            regularization parameter; (random) percentage of nodes dropped out
  61            of the training.
  62
  63        direct_link: boolean.
  64            indicates if the original predictors are included (True) in model's fitting or not (False).
  65
  66        n_clusters: int.
  67            number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering).
  68
  69        cluster_encode: bool.
  70            defines how the variable containing clusters is treated (default is one-hot)
  71            if `False`, then labels are used, without one-hot encoding.
  72
  73        type_clust: str.
  74            type of clustering method: currently k-means ('kmeans') or Gaussian
  75            Mixture Model ('gmm').
  76
  77        type_scaling: a tuple of 3 strings.
  78            scaling methods for inputs, hidden layer, and clustering respectively
  79            (and when relevant).
  80            Currently available: standardization ('std') or MinMax scaling ('minmax').
  81
  82        lags: int.
  83            number of lags used for each time series.
  84
  85        type_pi: str.
  86            type of prediction interval; currently:
  87            - "gaussian": simple, fast, but: assumes stationarity of Gaussian in-sample residuals and independence in the multivariate case
  88            - "kde": based on Kernel Density Estimation of in-sample residuals
  89            - "bootstrap": based on independent bootstrap of in-sample residuals
  90            - "block-bootstrap": based on basic block bootstrap of in-sample residuals
  91            - "scp-kde": Sequential split conformal prediction with Kernel Density Estimation of calibrated residuals
  92            - "scp-bootstrap": Sequential split conformal prediction with independent bootstrap of calibrated residuals
  93            - "scp-block-bootstrap": Sequential split conformal prediction with basic block bootstrap of calibrated residuals
  94            - "scp2-kde": Sequential split conformal prediction with Kernel Density Estimation of standardized calibrated residuals
  95            - "scp2-bootstrap": Sequential split conformal prediction with independent bootstrap of standardized calibrated residuals
  96            - "scp2-block-bootstrap": Sequential split conformal prediction with basic block bootstrap of standardized calibrated residuals
  97            - based on copulas of in-sample residuals: 'vine-tll', 'vine-bb1', 'vine-bb6', 'vine-bb7', 'vine-bb8', 'vine-clayton',
  98            'vine-frank', 'vine-gaussian', 'vine-gumbel', 'vine-indep', 'vine-joe', 'vine-student'
  99            - 'scp-vine-tll', 'scp-vine-bb1', 'scp-vine-bb6', 'scp-vine-bb7', 'scp-vine-bb8', 'scp-vine-clayton',
 100            'scp-vine-frank', 'scp-vine-gaussian', 'scp-vine-gumbel', 'scp-vine-indep', 'scp-vine-joe', 'scp-vine-student'
 101            - 'scp2-vine-tll', 'scp2-vine-bb1', 'scp2-vine-bb6', 'scp2-vine-bb7', 'scp2-vine-bb8', 'scp2-vine-clayton',
 102            'scp2-vine-frank', 'scp2-vine-gaussian', 'scp2-vine-gumbel', 'scp2-vine-indep', 'scp2-vine-joe', 'scp2-vine-student'
 103
 104        block_size: int.
 105            size of block for 'type_pi' in ("block-bootstrap", "scp-block-bootstrap", "scp2-block-bootstrap").
 106            Default is round(3.15*(n_residuals^1/3))
 107
 108        replications: int.
 109            number of replications (if needed, for predictive simulation). Default is 'None'.
 110
 111        kernel: str.
 112            the kernel to use for residuals density estimation (used for predictive simulation). Currently, either 'gaussian' or 'tophat'.
 113
 114        agg: str.
 115            either "mean" or "median" for simulation of bootstrap aggregating
 116
 117        seed: int.
 118            reproducibility seed for nodes_sim=='uniform' or predictive simulation.
 119
 120        backend: str.
 121            "cpu" or "gpu" or "tpu".
 122
 123        verbose: int.
 124            0: not printing; 1: printing
 125
 126        show_progress: bool.
 127            True: progress bar when fitting each series; False: no progress bar when fitting each series
 128
 129    Attributes:
 130
 131        fit_objs_: dict
 132            objects adjusted to each individual time series
 133
 134        y_: {array-like}
 135            MTS responses (most recent observations first)
 136
 137        X_: {array-like}
 138            MTS lags
 139
 140        xreg_: {array-like}
 141            external regressors
 142
 143        y_means_: dict
 144            a dictionary of each series mean values
 145
 146        preds_: {array-like}
 147            successive model predictions
 148
 149        preds_std_: {array-like}
 150            standard deviation around the predictions for Bayesian base learners (`obj`)
 151
 152        gaussian_preds_std_: {array-like}
 153            standard deviation around the predictions for `type_pi='gaussian'`
 154
 155        return_std_: boolean
 156            return uncertainty or not (set in predict)
 157
 158        df_: data frame
 159            the input data frame, in case a data.frame is provided to `fit`
 160
 161        n_obs_: int
 162            number of time series observations (number of rows for multivariate)
 163
 164        level_: int
 165            level of confidence for prediction intervals (default is 95)
 166
 167        residuals_: {array-like}
 168            in-sample residuals (for `type_pi` not conformal prediction) or calibrated residuals
 169            (for `type_pi` in conformal prediction)
 170
 171        residuals_sims_: tuple of {array-like}
 172            simulations of in-sample residuals (for `type_pi` not conformal prediction) or
 173            calibrated residuals (for `type_pi` in conformal prediction)
 174
 175        kde_: A scikit-learn object, see https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KernelDensity.html
 176
 177        residuals_std_dev_: residuals standard deviation
 178
 179    Examples:
 180
 181    Example 1:
 182
 183    ```python
 184    import nnetsauce as ns
 185    import numpy as np
 186    from sklearn import linear_model
 187    np.random.seed(123)
 188
 189    M = np.random.rand(10, 3)
 190    M[:,0] = 10*M[:,0]
 191    M[:,2] = 25*M[:,2]
 192    print(M)
 193
 194    # Adjust Bayesian Ridge
 195    regr4 = linear_model.BayesianRidge()
 196    obj_MTS = ns.MTS(regr4, lags = 1, n_hidden_features=5)
 197    obj_MTS.fit(M)
 198    print(obj_MTS.predict())
 199
 200    # with credible intervals
 201    print(obj_MTS.predict(return_std=True, level=80))
 202
 203    print(obj_MTS.predict(return_std=True, level=95))
 204    ```
 205
 206    Example 2:
 207
 208    ```python
 209    import nnetsauce as ns
 210    import numpy as np
 211    from sklearn import linear_model
 212
 213    dataset = {
 214    'date' : ['2001-01-01', '2002-01-01', '2003-01-01', '2004-01-01', '2005-01-01'],
 215    'series1' : [34, 30, 35.6, 33.3, 38.1],
 216    'series2' : [4, 5.5, 5.6, 6.3, 5.1],
 217    'series3' : [100, 100.5, 100.6, 100.2, 100.1]}
 218    df = pd.DataFrame(dataset).set_index('date')
 219    print(df)
 220
 221    # Adjust Bayesian Ridge
 222    regr5 = linear_model.BayesianRidge()
 223    obj_MTS = ns.MTS(regr5, lags = 1, n_hidden_features=5)
 224    obj_MTS.fit(df)
 225    print(obj_MTS.predict())
 226
 227    # with credible intervals
 228    print(obj_MTS.predict(return_std=True, level=80))
 229
 230    print(obj_MTS.predict(return_std=True, level=95))
 231    ```
 232    """
 233
 234    # construct the object -----
 235
 236    def __init__(
 237        self,
 238        obj,
 239        n_hidden_features=5,
 240        activation_name="relu",
 241        a=0.01,
 242        nodes_sim="sobol",
 243        bias=True,
 244        dropout=0,
 245        direct_link=True,
 246        n_clusters=2,
 247        cluster_encode=True,
 248        type_clust="kmeans",
 249        type_scaling=("std", "std", "std"),
 250        lags=1,
 251        type_pi="kde",
 252        block_size=None,
 253        replications=None,
 254        kernel="gaussian",
 255        agg="mean",
 256        seed=123,
 257        backend="cpu",
 258        verbose=0,
 259        show_progress=True,
 260    ):
 261        assert int(lags) == lags, "parameter 'lags' should be an integer"
 262
 263        super().__init__(
 264            n_hidden_features=n_hidden_features,
 265            activation_name=activation_name,
 266            a=a,
 267            nodes_sim=nodes_sim,
 268            bias=bias,
 269            dropout=dropout,
 270            direct_link=direct_link,
 271            n_clusters=n_clusters,
 272            cluster_encode=cluster_encode,
 273            type_clust=type_clust,
 274            type_scaling=type_scaling,
 275            seed=seed,
 276            backend=backend,
 277        )
 278
 279        self.obj = obj
 280        self.n_series = None
 281        self.lags = lags
 282        self.type_pi = type_pi
 283        self.block_size = block_size
 284        self.replications = replications
 285        self.kernel = kernel
 286        self.agg = agg
 287        self.verbose = verbose
 288        self.show_progress = show_progress
 289        self.series_names = None
 290        self.input_dates = None
 291        self.fit_objs_ = {}
 292        self.y_ = None  # MTS responses (most recent observations first)
 293        self.X_ = None  # MTS lags
 294        self.xreg_ = None
 295        self.y_means_ = {}
 296        self.mean_ = None
 297        self.upper_ = None
 298        self.lower_ = None
 299        self.output_dates_ = None
 300        self.preds_std_ = []
 301        self.gaussian_preds_std_ = None
 302        self.alpha_ = None
 303        self.return_std_ = None
 304        self.df_ = None
 305        self.residuals_ = []
 306        self.abs_calib_residuals_ = None
 307        self.calib_residuals_quantile_ = None
 308        self.residuals_sims_ = None
 309        self.kde_ = None
 310        self.sims_ = None
 311        self.residuals_std_dev_ = None
 312        self.n_obs_ = None
 313        self.level_ = None
 314
 315    def fit(self, X, xreg=None, **kwargs):
 316        """Fit MTS model to training data X, with optional regressors xreg
 317
 318        Parameters:
 319
 320        X: {array-like}, shape = [n_samples, n_features]
 321            Training time series, where n_samples is the number
 322            of samples and n_features is the number of features;
 323            X must be in increasing order (most recent observations last)
 324
 325        xreg: {array-like}, shape = [n_samples, n_features_xreg]
 326            Additional (external) regressors to be passed to self.obj
 327            xreg must be in 'increasing' order (most recent observations last)
 328
 329        **kwargs: for now, additional parameters to be passed to for kernel density estimation, when needed (see sklearn.neighbors.KernelDensity)
 330
 331        Returns:
 332
 333        self: object
 334        """
 335
 336        if (
 337            isinstance(X, pd.DataFrame) is False
 338        ):  # input data set is a numpy array
 339
 340            if xreg is None:
 341                X = pd.DataFrame(X)
 342                self.series_names = [
 343                    "series" + str(i) for i in range(X.shape[1])
 344                ]
 345            else:  # xreg is not None
 346                X = mo.cbind(X, xreg)
 347                self.xreg_ = xreg
 348
 349        else:  # input data set is a DataFrame with column names
 350
 351            X_index = None
 352            if X.index is not None:
 353                X_index = X.index
 354            if xreg is None:
 355                X = copy.deepcopy(mo.convert_df_to_numeric(X))
 356            else:
 357                X = copy.deepcopy(mo.cbind(mo.convert_df_to_numeric(X), xreg))
 358                self.xreg_ = xreg
 359            if X_index is not None:
 360                X.index = X_index
 361            self.series_names = X.columns.tolist()
 362
 363        if isinstance(X, pd.DataFrame):
 364            self.df_ = X
 365            X = X.values
 366            self.df_.columns = self.series_names
 367            self.input_dates = ts.compute_input_dates(self.df_)
 368        else:
 369            self.df_ = pd.DataFrame(X, columns=self.series_names)
 370            self.input_dates = ts.compute_input_dates(self.df_)
 371
 372        try:
 373            # multivariate time series
 374            n, p = X.shape
 375        except:
 376            # univariate time series
 377            n = X.shape[0]
 378            p = 1
 379
 380        self.n_obs_ = n
 381
 382        rep_1_n = np.repeat(1, n)
 383
 384        self.y_ = None
 385        self.X_ = None
 386        self.n_series = p
 387        self.fit_objs_.clear()
 388        self.y_means_.clear()
 389        residuals_ = []
 390        self.residuals_ = None
 391        self.residuals_sims_ = None
 392        self.kde_ = None
 393        self.sims_ = None
 394        self.scaled_Z_ = None
 395        self.centered_y_is_ = []
 396
 397        if p > 1:
 398            # multivariate time series
 399            mts_input = ts.create_train_inputs(X[::-1], self.lags)
 400        else:
 401            # univariate time series
 402            mts_input = ts.create_train_inputs(
 403                X.reshape(-1, 1)[::-1], self.lags
 404            )
 405
 406        self.y_ = mts_input[0]
 407
 408        self.X_ = mts_input[1]
 409
 410        dummy_y, scaled_Z = self.cook_training_set(y=rep_1_n, X=self.X_)
 411
 412        self.scaled_Z_ = scaled_Z
 413
 414        # loop on all the time series and adjust self.obj.fit
 415        if self.verbose > 0:
 416            print(
 417                f"\n Adjusting {type(self.obj).__name__} to multivariate time series... \n "
 418            )
 419
 420        if self.show_progress is True:
 421            iterator = tqdm(range(p))
 422        else:
 423            iterator = range(p)
 424
 425        if self.type_pi in (
 426            "gaussian",
 427            "kde",
 428            "bootstrap",
 429            "block-bootstrap",
 430        ) or self.type_pi.startswith("vine"):
 431            for i in iterator:
 432                y_mean = np.mean(self.y_[:, i])
 433                self.y_means_[i] = y_mean
 434                centered_y_i = self.y_[:, i] - y_mean
 435                self.centered_y_is_.append(centered_y_i)
 436                self.obj.fit(X=scaled_Z, y=centered_y_i)
 437                self.fit_objs_[i] = deepcopy(self.obj)
 438                residuals_.append(
 439                    (
 440                        centered_y_i - self.fit_objs_[i].predict(scaled_Z)
 441                    ).tolist()
 442                )
 443
 444        if self.type_pi.startswith("scp"):
 445            # split conformal prediction
 446            for i in iterator:
 447                n_y = self.y_.shape[0]
 448                n_y_half = n_y // 2
 449                first_half_idx = range(0, n_y_half)
 450                second_half_idx = range(n_y_half, n_y)
 451                y_mean_temp = np.mean(self.y_[first_half_idx, i])
 452                centered_y_i_temp = self.y_[first_half_idx, i] - y_mean_temp
 453                self.obj.fit(X=scaled_Z[first_half_idx, :], y=centered_y_i_temp)
 454                # calibrated residuals actually
 455                residuals_.append(
 456                    (
 457                        self.y_[second_half_idx, i]
 458                        - (
 459                            y_mean_temp
 460                            + self.obj.predict(scaled_Z[second_half_idx, :])
 461                        )
 462                    ).tolist()
 463                )
 464                # fit on the second half
 465                y_mean = np.mean(self.y_[second_half_idx, i])
 466                self.y_means_[i] = y_mean
 467                centered_y_i = self.y_[second_half_idx, i] - y_mean
 468                self.obj.fit(X=scaled_Z[second_half_idx, :], y=centered_y_i)
 469                self.fit_objs_[i] = deepcopy(self.obj)
 470
 471        self.residuals_ = np.asarray(residuals_).T
 472
 473        if self.type_pi == "gaussian":
 474            self.gaussian_preds_std_ = np.std(self.residuals_, axis=0)
 475
 476        if self.type_pi.startswith("scp2"):
 477            # Calculate mean and standard deviation for each column
 478            data_mean = np.mean(self.residuals_, axis=0)
 479            self.residuals_std_dev_ = np.std(self.residuals_, axis=0)
 480            # Center and scale the array using broadcasting
 481            self.residuals_ = (
 482                self.residuals_ - data_mean[np.newaxis, :]
 483            ) / self.residuals_std_dev_[np.newaxis, :]
 484
 485        if self.replications != None and "kde" in self.type_pi:
 486            if self.verbose > 0:
 487                print(f"\n Simulate residuals using {self.kernel} kernel... \n")
 488            assert self.kernel in (
 489                "gaussian",
 490                "tophat",
 491            ), "currently, 'kernel' must be either 'gaussian' or 'tophat'"
 492            kernel_bandwidths = {"bandwidth": np.logspace(-6, 6, 150)}
 493            grid = GridSearchCV(
 494                KernelDensity(kernel=self.kernel, **kwargs),
 495                param_grid=kernel_bandwidths,
 496            )
 497            grid.fit(self.residuals_)
 498
 499            if self.verbose > 0:
 500                print(
 501                    f"\n Best parameters for {self.kernel} kernel: {grid.best_params_} \n"
 502                )
 503
 504            self.kde_ = grid.best_estimator_
 505
 506        return self
 507
 508    def predict(self, h=5, level=95, **kwargs):
 509        """Forecast all the time series, h steps ahead
 510
 511        Parameters:
 512
 513        h: {integer}
 514            Forecasting horizon
 515
 516        level: {integer}
 517            Level of confidence (if obj has option 'return_std' and the
 518            posterior is gaussian)
 519
 520        new_xreg: {array-like}, shape = [n_samples = h, n_new_xreg]
 521            New values of additional (deterministic) regressors on horizon = h
 522            new_xreg must be in increasing order (most recent observations last)
 523
 524        **kwargs: additional parameters to be passed to
 525                self.cook_test_set
 526
 527        Returns:
 528
 529        model predictions for horizon = h: {array-like}, data frame or tuple.
 530        Standard deviation and prediction intervals are returned when
 531        `obj.predict` can return standard deviation
 532
 533        """
 534
 535        self.output_dates_, frequency = ts.compute_output_dates(self.df_, h)
 536
 537        self.level_ = level
 538
 539        self.return_std_ = False  # do not remove (/!\)
 540
 541        self.mean_ = None  # do not remove (/!\)
 542
 543        self.mean_ = deepcopy(self.y_)  # do not remove (/!\)
 544
 545        self.lower_ = None  # do not remove (/!\)
 546
 547        self.upper_ = None  # do not remove (/!\)
 548
 549        self.sims_ = None  # do not remove (/!\)
 550
 551        y_means_ = np.asarray([self.y_means_[i] for i in range(self.n_series)])
 552
 553        n_features = self.n_series * self.lags
 554
 555        self.alpha_ = 100 - level
 556
 557        pi_multiplier = norm.ppf(1 - self.alpha_ / 200)
 558
 559        if "return_std" in kwargs:  # bayesian forecasting
 560            self.return_std_ = True
 561            self.preds_std_ = []
 562            DescribeResult = namedtuple(
 563                "DescribeResult", ("mean", "lower", "upper")
 564            )  # to be updated
 565
 566        if "return_pi" in kwargs:  # split conformal, without simulation
 567            mean_pi_ = []
 568            lower_pi_ = []
 569            upper_pi_ = []
 570            DescribeResult = namedtuple(
 571                "DescribeResult", ("mean", "lower", "upper")
 572            )  # to be updated
 573
 574        if self.kde_ != None and "kde" in self.type_pi:  # kde
 575            if self.verbose == 1:
 576                self.residuals_sims_ = tuple(
 577                    self.kde_.sample(
 578                        n_samples=h, random_state=self.seed + 100 * i
 579                    )
 580                    for i in tqdm(range(self.replications))
 581                )
 582            elif self.verbose == 0:
 583                self.residuals_sims_ = tuple(
 584                    self.kde_.sample(
 585                        n_samples=h, random_state=self.seed + 100 * i
 586                    )
 587                    for i in range(self.replications)
 588                )
 589
 590        if self.type_pi in ("bootstrap", "scp-bootstrap", "scp2-bootstrap"):
 591            assert self.replications is not None and isinstance(
 592                self.replications, int
 593            ), "'replications' must be provided and be an integer"
 594            if self.verbose == 1:
 595                self.residuals_sims_ = tuple(
 596                    ts.bootstrap(
 597                        self.residuals_,
 598                        h=h,
 599                        block_size=None,
 600                        seed=self.seed + 100 * i,
 601                    )
 602                    for i in tqdm(range(self.replications))
 603                )
 604            elif self.verbose == 0:
 605                self.residuals_sims_ = tuple(
 606                    ts.bootstrap(
 607                        self.residuals_,
 608                        h=h,
 609                        block_size=None,
 610                        seed=self.seed + 100 * i,
 611                    )
 612                    for i in range(self.replications)
 613                )
 614
 615        if self.type_pi in (
 616            "block-bootstrap",
 617            "scp-block-bootstrap",
 618            "scp2-block-bootstrap",
 619        ):
 620            if self.block_size is None:
 621                self.block_size = int(
 622                    np.ceil(3.15 * (self.residuals_.shape[0] ** (1 / 3)))
 623                )
 624
 625            assert self.replications is not None and isinstance(
 626                self.replications, int
 627            ), "'replications' must be provided and be an integer"
 628            if self.verbose == 1:
 629                self.residuals_sims_ = tuple(
 630                    ts.bootstrap(
 631                        self.residuals_,
 632                        h=h,
 633                        block_size=self.block_size,
 634                        seed=self.seed + 100 * i,
 635                    )
 636                    for i in tqdm(range(self.replications))
 637                )
 638            elif self.verbose == 0:
 639                self.residuals_sims_ = tuple(
 640                    ts.bootstrap(
 641                        self.residuals_,
 642                        h=h,
 643                        block_size=self.block_size,
 644                        seed=self.seed + 100 * i,
 645                    )
 646                    for i in range(self.replications)
 647                )
 648
 649        if "vine" in self.type_pi:
 650            if self.verbose == 1:
 651                self.residuals_sims_ = tuple(
 652                    vinecopula_sample(
 653                        x=self.residuals_,
 654                        n_samples=h,
 655                        method=self.type_pi,
 656                        random_state=self.seed + 100 * i,
 657                    )
 658                    for i in tqdm(range(self.replications))
 659                )
 660            elif self.verbose == 0:
 661                self.residuals_sims_ = tuple(
 662                    vinecopula_sample(
 663                        x=self.residuals_,
 664                        n_samples=h,
 665                        method=self.type_pi,
 666                        random_state=self.seed + 100 * i,
 667                    )
 668                    for i in range(self.replications)
 669                )
 670
 671        for _ in range(h):
 672
 673            new_obs = ts.reformat_response(self.mean_, self.lags)
 674
 675            new_X = new_obs.reshape(1, n_features)
 676
 677            cooked_new_X = self.cook_test_set(new_X, **kwargs)
 678
 679            if "return_std" in kwargs:
 680                self.preds_std_.append(
 681                    [
 682                        np.asarray(
 683                            self.fit_objs_[i].predict(
 684                                cooked_new_X, return_std=True
 685                            )[1]
 686                        ).item()
 687                        for i in range(self.n_series)
 688                    ]
 689                )
 690
 691            if "return_pi" in kwargs:
 692                for i in range(self.n_series):
 693                    preds_pi = self.fit_objs_[i].predict(
 694                        cooked_new_X, return_pi=True
 695                    )
 696                    mean_pi_.append(preds_pi.mean[0])
 697                    lower_pi_.append(preds_pi.lower[0])
 698                    upper_pi_.append(preds_pi.upper[0])
 699
 700            predicted_cooked_new_X = np.asarray(
 701                [
 702                    np.asarray(self.fit_objs_[i].predict(cooked_new_X)).item()
 703                    for i in range(self.n_series)
 704                ]
 705            )
 706
 707            preds = np.asarray(y_means_ + predicted_cooked_new_X)
 708
 709            self.mean_ = mo.rbind(preds, self.mean_)  # preallocate?
 710
 711        # function's return ----------------------------------------------------------------------
 712        self.mean_ = pd.DataFrame(
 713            self.mean_[0:h, :][::-1],
 714            columns=self.df_.columns,
 715            index=self.output_dates_,
 716        )
 717
 718        if (
 719            (("return_std" not in kwargs) and ("return_pi" not in kwargs))
 720            and (self.type_pi not in ("gaussian", "scp"))
 721        ) or ("vine" in self.type_pi):
 722
 723            if self.replications is None:
 724                return self.mean_
 725
 726            # if "return_std" not in kwargs and self.replications is not None
 727            meanf = []
 728            lower = []
 729            upper = []
 730
 731            if "scp2" in self.type_pi:
 732
 733                if self.verbose == 1:
 734                    self.sims_ = tuple(
 735                        (
 736                            self.mean_
 737                            + self.residuals_sims_[i]
 738                            * self.residuals_std_dev_[np.newaxis, :]
 739                            for i in tqdm(range(self.replications))
 740                        )
 741                    )
 742                elif self.verbose == 0:
 743                    self.sims_ = tuple(
 744                        (
 745                            self.mean_
 746                            + self.residuals_sims_[i]
 747                            * self.residuals_std_dev_[np.newaxis, :]
 748                            for i in range(self.replications)
 749                        )
 750                    )
 751            else:
 752
 753                if self.verbose == 1:
 754                    self.sims_ = tuple(
 755                        (
 756                            self.mean_ + self.residuals_sims_[i]
 757                            for i in tqdm(range(self.replications))
 758                        )
 759                    )
 760                elif self.verbose == 0:
 761                    self.sims_ = tuple(
 762                        (
 763                            self.mean_ + self.residuals_sims_[i]
 764                            for i in range(self.replications)
 765                        )
 766                    )
 767
 768            DescribeResult = namedtuple(
 769                "DescribeResult", ("mean", "sims", "lower", "upper")
 770            )
 771            for ix in range(self.n_series):
 772                sims_ix = getsims(self.sims_, ix)
 773                if self.agg == "mean":
 774                    meanf.append(np.mean(sims_ix, axis=1))
 775                else:
 776                    meanf.append(np.median(sims_ix, axis=1))
 777                lower.append(np.quantile(sims_ix, q=self.alpha_ / 200, axis=1))
 778                upper.append(
 779                    np.quantile(sims_ix, q=1 - self.alpha_ / 200, axis=1)
 780                )
 781
 782            self.mean_ = pd.DataFrame(
 783                np.asarray(meanf).T,
 784                columns=self.series_names,  # self.df_.columns,
 785                index=self.output_dates_,
 786            )
 787
 788            self.lower_ = pd.DataFrame(
 789                np.asarray(lower).T,
 790                columns=self.series_names,  # self.df_.columns,
 791                index=self.output_dates_,
 792            )
 793
 794            self.upper_ = pd.DataFrame(
 795                np.asarray(upper).T,
 796                columns=self.series_names,  # self.df_.columns,
 797                index=self.output_dates_,
 798            )
 799
 800            res = DescribeResult(
 801                self.mean_, self.sims_, self.lower_, self.upper_
 802            )
 803
 804            if self.xreg_ is not None:
 805
 806                if len(self.xreg_.shape) > 1:
 807
 808                    res2 = mx.tuple_map(
 809                        res,
 810                        lambda x: mo.delete_last_columns(
 811                            x, num_columns=self.xreg_.shape[1]
 812                        ),
 813                    )
 814
 815                else:
 816
 817                    res2 = mx.tuple_map(
 818                        res, lambda x: mo.delete_last_columns(x, num_columns=1)
 819                    )
 820
 821                return res2
 822
 823            else:
 824
 825                return res
 826
 827        if (
 828            (("return_std" in kwargs) or ("return_pi" in kwargs))
 829            and (self.type_pi not in ("gaussian", "scp"))
 830        ) or "vine" in self.type_pi:
 831            DescribeResult = namedtuple(
 832                "DescribeResult", ("mean", "lower", "upper")
 833            )
 834
 835            self.mean_ = pd.DataFrame(
 836                np.asarray(self.mean_),
 837                columns=self.series_names,  # self.df_.columns,
 838                index=self.output_dates_,
 839            )
 840
 841            if "return_std" in kwargs:
 842
 843                self.preds_std_ = np.asarray(self.preds_std_)
 844
 845                self.lower_ = pd.DataFrame(
 846                    self.mean_.values - pi_multiplier * self.preds_std_,
 847                    columns=self.series_names,  # self.df_.columns,
 848                    index=self.output_dates_,
 849                )
 850
 851                self.upper_ = pd.DataFrame(
 852                    self.mean_.values + pi_multiplier * self.preds_std_,
 853                    columns=self.series_names,  # self.df_.columns,
 854                    index=self.output_dates_,
 855                )
 856
 857            if "return_pi" in kwargs:
 858
 859                self.lower_ = pd.DataFrame(
 860                    np.asarray(lower_pi_).reshape(h, self.n_series)
 861                    + y_means_[np.newaxis, :],
 862                    columns=self.series_names,  # self.df_.columns,
 863                    index=self.output_dates_,
 864                )
 865
 866                self.upper_ = pd.DataFrame(
 867                    np.asarray(upper_pi_).reshape(h, self.n_series)
 868                    + y_means_[np.newaxis, :],
 869                    columns=self.series_names,  # self.df_.columns,
 870                    index=self.output_dates_,
 871                )
 872
 873            res = DescribeResult(self.mean_, self.lower_, self.upper_)
 874
 875            if self.xreg_ is not None:
 876                if len(self.xreg_.shape) > 1:
 877                    res2 = mx.tuple_map(
 878                        res,
 879                        lambda x: mo.delete_last_columns(
 880                            x, num_columns=self.xreg_.shape[1]
 881                        ),
 882                    )
 883                else:
 884                    res2 = mx.tuple_map(
 885                        res, lambda x: mo.delete_last_columns(x, num_columns=1)
 886                    )
 887                return DescribeResult(res2[0], res2[1], res2[2])
 888
 889            return res
 890
 891        if self.type_pi == "gaussian":
 892
 893            DescribeResult = namedtuple(
 894                "DescribeResult", ("mean", "lower", "upper")
 895            )
 896
 897            self.mean_ = pd.DataFrame(
 898                np.asarray(self.mean_),
 899                columns=self.series_names,  # self.df_.columns,
 900                index=self.output_dates_,
 901            )
 902
 903            self.lower_ = pd.DataFrame(
 904                self.mean_.values - pi_multiplier * self.gaussian_preds_std_,
 905                columns=self.series_names,  # self.df_.columns,
 906                index=self.output_dates_,
 907            )
 908
 909            self.upper_ = pd.DataFrame(
 910                self.mean_.values + pi_multiplier * self.gaussian_preds_std_,
 911                columns=self.series_names,  # self.df_.columns,
 912                index=self.output_dates_,
 913            )
 914
 915            res = DescribeResult(self.mean_, self.lower_, self.upper_)
 916
 917            if self.xreg_ is not None:
 918                if len(self.xreg_.shape) > 1:
 919                    res2 = mx.tuple_map(
 920                        res,
 921                        lambda x: mo.delete_last_columns(
 922                            x, num_columns=self.xreg_.shape[1]
 923                        ),
 924                    )
 925                else:
 926                    res2 = mx.tuple_map(
 927                        res, lambda x: mo.delete_last_columns(x, num_columns=1)
 928                    )
 929                return DescribeResult(res2[0], res2[1], res2[2])
 930
 931            return res
 932
 933    def score(self, X, training_index, testing_index, scoring=None, **kwargs):
 934        """Train on training_index, score on testing_index."""
 935
 936        assert (
 937            bool(set(training_index).intersection(set(testing_index))) == False
 938        ), "Non-overlapping 'training_index' and 'testing_index' required"
 939
 940        # Dimensions
 941        try:
 942            # multivariate time series
 943            n, p = X.shape
 944        except:
 945            # univariate time series
 946            n = X.shape[0]
 947            p = 1
 948
 949        # Training and testing sets
 950        if p > 1:
 951            X_train = X[training_index, :]
 952            X_test = X[testing_index, :]
 953        else:
 954            X_train = X[training_index]
 955            X_test = X[testing_index]
 956
 957        # Horizon
 958        h = len(testing_index)
 959        assert (
 960            len(training_index) + h
 961        ) <= n, "Please check lengths of training and testing windows"
 962
 963        # Fit and predict
 964        self.fit(X_train, **kwargs)
 965        preds = self.predict(h=h, **kwargs)
 966
 967        if scoring is None:
 968            scoring = "neg_root_mean_squared_error"
 969
 970        # check inputs
 971        assert scoring in (
 972            "explained_variance",
 973            "neg_mean_absolute_error",
 974            "neg_mean_squared_error",
 975            "neg_root_mean_squared_error",
 976            "neg_mean_squared_log_error",
 977            "neg_median_absolute_error",
 978            "r2",
 979        ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
 980                               'neg_mean_squared_error', 'neg_root_mean_squared_error', 'neg_mean_squared_log_error', \
 981                               'neg_median_absolute_error', 'r2')"
 982
 983        scoring_options = {
 984            "explained_variance": skm2.explained_variance_score,
 985            "neg_mean_absolute_error": skm2.mean_absolute_error,
 986            "neg_mean_squared_error": skm2.mean_squared_error,
 987            "neg_root_mean_squared_error": lambda x, y: np.sqrt(
 988                skm2.mean_squared_error(x, y)
 989            ),
 990            "neg_mean_squared_log_error": skm2.mean_squared_log_error,
 991            "neg_median_absolute_error": skm2.median_absolute_error,
 992            "r2": skm2.r2_score,
 993        }
 994
 995        # if p > 1:
 996        #     return tuple(
 997        #         [
 998        #             scoring_options[scoring](
 999        #                 X_test[:, i], preds[:, i]#, **kwargs
1000        #             )
1001        #             for i in range(p)
1002        #         ]
1003        #     )
1004        # else:
1005        return scoring_options[scoring](X_test, preds)
1006
1007    def plot(self, series=None, type_axis="dates", type_plot="pi"):
1008        """Plot time series forecast
1009
1010        Parameters:
1011
1012        series: {integer} or {string}
1013            series index or name
1014
1015        """
1016
1017        assert all(
1018            [
1019                self.mean_ is not None,
1020                self.lower_ is not None,
1021                self.upper_ is not None,
1022                self.output_dates_ is not None,
1023            ]
1024        ), "model forecasting must be obtained first (with predict)"
1025
1026        if series is None:
1027            assert (
1028                self.n_series == 1
1029            ), "please specify series index or name (n_series > 1)"
1030            series = 0
1031
1032        if isinstance(series, str):
1033            assert (
1034                series in self.series_names
1035            ), f"series {series} doesn't exist in the input dataset"
1036            series_idx = self.df_.columns.get_loc(series)
1037        else:
1038            assert isinstance(series, int) and (
1039                0 <= series < self.n_series
1040            ), f"check series index (< {self.n_series})"
1041            series_idx = series
1042
1043        y_all = list(self.df_.iloc[:, series_idx]) + list(
1044            self.mean_.iloc[:, series_idx]
1045        )
1046        y_test = list(self.mean_.iloc[:, series_idx])
1047        n_points_all = len(y_all)
1048        n_points_train = self.df_.shape[0]
1049
1050        if type_axis == "numeric":
1051            x_all = [i for i in range(n_points_all)]
1052            x_test = [i for i in range(n_points_train, n_points_all)]
1053
1054        if type_axis == "dates":  # use dates
1055            x_all = np.concatenate(
1056                (self.input_dates.values, self.output_dates_.values), axis=None
1057            )
1058            x_test = self.output_dates_.values
1059
1060        if type_plot == "pi":
1061            fig, ax = plt.subplots()
1062            ax.plot(x_all, y_all, "-")
1063            ax.plot(x_test, y_test, "-", color="orange")
1064            ax.fill_between(
1065                x_test,
1066                self.lower_.iloc[:, series_idx],
1067                self.upper_.iloc[:, series_idx],
1068                alpha=0.2,
1069                color="orange",
1070            )
1071            if self.replications is None:
1072                if self.n_series > 1:
1073                    plt.title(
1074                        f"prediction intervals for {series}",
1075                        loc="left",
1076                        fontsize=12,
1077                        fontweight=0,
1078                        color="black",
1079                    )
1080                else:
1081                    plt.title(
1082                        f"prediction intervals for input time series",
1083                        loc="left",
1084                        fontsize=12,
1085                        fontweight=0,
1086                        color="black",
1087                    )
1088                plt.show()
1089            else:  # self.replications is not None
1090                if self.n_series > 1:
1091                    plt.title(
1092                        f"prediction intervals for {self.replications} simulations of {series}",
1093                        loc="left",
1094                        fontsize=12,
1095                        fontweight=0,
1096                        color="black",
1097                    )
1098                else:
1099                    plt.title(
1100                        f"prediction intervals for {self.replications} simulations of input time series",
1101                        loc="left",
1102                        fontsize=12,
1103                        fontweight=0,
1104                        color="black",
1105                    )
1106                plt.show()
1107
1108        if type_plot == "spaghetti":
1109            palette = plt.get_cmap("Set1")
1110            sims_ix = getsims(self.sims_, series_idx)
1111            plt.plot(x_all, y_all, "-")
1112            for col_ix in range(
1113                sims_ix.shape[1]
1114            ):  # avoid this when there are thousands of simulations
1115                plt.plot(
1116                    x_test,
1117                    sims_ix[:, col_ix],
1118                    "-",
1119                    color=palette(col_ix),
1120                    linewidth=1,
1121                    alpha=0.9,
1122                )
1123            plt.plot(x_all, y_all, "-", color="black")
1124            plt.plot(x_test, y_test, "-", color="blue")
1125            # Add titles
1126            if self.n_series > 1:
1127                plt.title(
1128                    f"{self.replications} simulations of {series}",
1129                    loc="left",
1130                    fontsize=12,
1131                    fontweight=0,
1132                    color="black",
1133                )
1134            else:
1135                plt.title(
1136                    f"{self.replications} simulations of input time series",
1137                    loc="left",
1138                    fontsize=12,
1139                    fontweight=0,
1140                    color="black",
1141                )
1142            plt.xlabel("Time")
1143            plt.ylabel("Values")
1144            # Show the graph
1145            plt.show()
1146
1147    def cross_val_score(
1148        self,
1149        X,
1150        scoring="root_mean_squared_error",
1151        n_jobs=None,
1152        verbose=0,
1153        xreg=None,
1154        initial_window=5,
1155        horizon=3,
1156        fixed_window=False,
1157        show_progress=True,
1158        level=95,
1159        **kwargs,
1160    ):
1161        """Evaluate a score by time series cross-validation.
1162
1163        Parameters:
1164
1165            X: {array-like, sparse matrix} of shape (n_samples, n_features)
1166                The data to fit.
1167
1168            scoring: str or a function
1169                A str in ('root_mean_squared_error', 'mean_squared_error', 'mean_error',
1170                'mean_absolute_error', 'mean_error', 'mean_percentage_error',
1171                'mean_absolute_percentage_error',  'winkler_score', 'coverage')
1172                Or a function defined as 'coverage' and 'winkler_score' in `utils.timeseries`
1173
1174            n_jobs: int, default=None
1175                Number of jobs to run in parallel.
1176
1177            verbose: int, default=0
1178                The verbosity level.
1179
1180            xreg: array-like, optional (default=None)
1181                Additional (external) regressors to be passed to `fit`
1182                xreg must be in 'increasing' order (most recent observations last)
1183
1184            initial_window: int
1185                initial number of consecutive values in each training set sample
1186
1187            horizon: int
1188                number of consecutive values in test set sample
1189
1190            fixed_window: boolean
1191                if False, all training samples start at index 0, and the training
1192                window's size is increasing.
1193                if True, the training window's size is fixed, and the window is
1194                rolling forward
1195
1196            show_progress: boolean
1197                if True, a progress bar is printed
1198
1199            **kwargs: dict
1200                additional parameters to be passed to `fit` and `predict`
1201
1202        Returns:
1203
1204            A tuple: descriptive statistics or errors and raw errors
1205
1206        """
1207        tscv = TimeSeriesSplit()
1208
1209        tscv_obj = tscv.split(
1210            X,
1211            initial_window=initial_window,
1212            horizon=horizon,
1213            fixed_window=fixed_window,
1214        )
1215
1216        if isinstance(scoring, str):
1217
1218            assert scoring in (
1219                "root_mean_squared_error",
1220                "mean_squared_error",
1221                "mean_error",
1222                "mean_absolute_error",
1223                "mean_percentage_error",
1224                "mean_absolute_percentage_error",
1225                "winkler_score",
1226                "coverage",
1227            ), "must have scoring in ('root_mean_squared_error', 'mean_squared_error', 'mean_error', 'mean_absolute_error', 'mean_error', 'mean_percentage_error', 'mean_absolute_percentage_error',  'winkler_score', 'coverage')"
1228
1229            def err_func(X_test, X_pred, scoring):
1230                if (self.replications is not None) or (
1231                    self.type_pi == "gaussian"
1232                ):  # probabilistic
1233                    if scoring == "winkler_score":
1234                        return winkler_score(X_pred, X_test, level=level)
1235                    elif scoring == "coverage":
1236                        return coverage(X_pred, X_test, level=level)
1237                    else:
1238                        return mean_errors(
1239                            pred=X_pred.mean, actual=X_test, scoring=scoring
1240                        )
1241                else:  # not probabilistic
1242                    return mean_errors(
1243                        pred=X_pred, actual=X_test, scoring=scoring
1244                    )
1245
1246        else:  # isinstance(scoring, str) = False
1247
1248            err_func = scoring
1249
1250        errors = []
1251
1252        train_indices = []
1253
1254        test_indices = []
1255
1256        for train_index, test_index in tscv_obj:
1257            train_indices.append(train_index)
1258            test_indices.append(test_index)
1259
1260        if show_progress is True:
1261            iterator = tqdm(
1262                zip(train_indices, test_indices), total=len(train_indices)
1263            )
1264        else:
1265            iterator = zip(train_indices, test_indices)
1266
1267        for train_index, test_index in iterator:
1268
1269            if verbose == 1:
1270                print(f"TRAIN: {train_index}")
1271                print(f"TEST: {test_index}")
1272
1273            if isinstance(X, pd.DataFrame):
1274                self.fit(X.iloc[train_index, :], xreg=xreg, **kwargs)
1275                X_test = X.iloc[test_index, :]
1276            else:
1277                self.fit(X[train_index, :], xreg=xreg, **kwargs)
1278                X_test = X[test_index, :]
1279            X_pred = self.predict(h=int(len(test_index)), level=level, **kwargs)
1280
1281            errors.append(err_func(X_test, X_pred, scoring))
1282
1283        res = np.asarray(errors)
1284
1285        return res, describe(res)

Univariate and multivariate time series (MTS) forecasting with Quasi-Randomized networks

Parameters:

obj: object.
    any object containing a method fit (obj.fit()) and a method predict
    (obj.predict()).

n_hidden_features: int.
    number of nodes in the hidden layer.

activation_name: str.
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'.

a: float.
    hyperparameter for 'prelu' or 'elu' activation function.

nodes_sim: str.
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'.

bias: boolean.
    indicates if the hidden layer contains a bias term (True) or not
    (False).

dropout: float.
    regularization parameter; (random) percentage of nodes dropped out
    of the training.

direct_link: boolean.
    indicates if the original predictors are included (True) in model's fitting or not (False).

n_clusters: int.
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering).

cluster_encode: bool.
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding.

type_clust: str.
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm').

type_scaling: a tuple of 3 strings.
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax').

lags: int.
    number of lags used for each time series.

type_pi: str.
    type of prediction interval; currently:
    - "gaussian": simple, fast, but: assumes stationarity of Gaussian in-sample residuals and independence in the multivariate case
    - "kde": based on Kernel Density Estimation of in-sample residuals
    - "bootstrap": based on independent bootstrap of in-sample residuals
    - "block-bootstrap": based on basic block bootstrap of in-sample residuals
    - "scp-kde": Sequential split conformal prediction with Kernel Density Estimation of calibrated residuals
    - "scp-bootstrap": Sequential split conformal prediction with independent bootstrap of calibrated residuals
    - "scp-block-bootstrap": Sequential split conformal prediction with basic block bootstrap of calibrated residuals
    - "scp2-kde": Sequential split conformal prediction with Kernel Density Estimation of standardized calibrated residuals
    - "scp2-bootstrap": Sequential split conformal prediction with independent bootstrap of standardized calibrated residuals
    - "scp2-block-bootstrap": Sequential split conformal prediction with basic block bootstrap of standardized calibrated residuals
    - based on copulas of in-sample residuals: 'vine-tll', 'vine-bb1', 'vine-bb6', 'vine-bb7', 'vine-bb8', 'vine-clayton',
    'vine-frank', 'vine-gaussian', 'vine-gumbel', 'vine-indep', 'vine-joe', 'vine-student'
    - 'scp-vine-tll', 'scp-vine-bb1', 'scp-vine-bb6', 'scp-vine-bb7', 'scp-vine-bb8', 'scp-vine-clayton',
    'scp-vine-frank', 'scp-vine-gaussian', 'scp-vine-gumbel', 'scp-vine-indep', 'scp-vine-joe', 'scp-vine-student'
    - 'scp2-vine-tll', 'scp2-vine-bb1', 'scp2-vine-bb6', 'scp2-vine-bb7', 'scp2-vine-bb8', 'scp2-vine-clayton',
    'scp2-vine-frank', 'scp2-vine-gaussian', 'scp2-vine-gumbel', 'scp2-vine-indep', 'scp2-vine-joe', 'scp2-vine-student'

block_size: int.
    size of block for 'type_pi' in ("block-bootstrap", "scp-block-bootstrap", "scp2-block-bootstrap").
    Default is round(3.15*(n_residuals^1/3))

replications: int.
    number of replications (if needed, for predictive simulation). Default is 'None'.

kernel: str.
    the kernel to use for residuals density estimation (used for predictive simulation). Currently, either 'gaussian' or 'tophat'.

agg: str.
    either "mean" or "median" for simulation of bootstrap aggregating

seed: int.
    reproducibility seed for nodes_sim=='uniform' or predictive simulation.

backend: str.
    "cpu" or "gpu" or "tpu".

verbose: int.
    0: not printing; 1: printing

show_progress: bool.
    True: progress bar when fitting each series; False: no progress bar when fitting each series

Attributes:

fit_objs_: dict
    objects adjusted to each individual time series

y_: {array-like}
    MTS responses (most recent observations first)

X_: {array-like}
    MTS lags

xreg_: {array-like}
    external regressors

y_means_: dict
    a dictionary of each series mean values

preds_: {array-like}
    successive model predictions

preds_std_: {array-like}
    standard deviation around the predictions for Bayesian base learners (`obj`)

gaussian_preds_std_: {array-like}
    standard deviation around the predictions for `type_pi='gaussian'`

return_std_: boolean
    return uncertainty or not (set in predict)

df_: data frame
    the input data frame, in case a data.frame is provided to `fit`

n_obs_: int
    number of time series observations (number of rows for multivariate)

level_: int
    level of confidence for prediction intervals (default is 95)

residuals_: {array-like}
    in-sample residuals (for `type_pi` not conformal prediction) or calibrated residuals
    (for `type_pi` in conformal prediction)

residuals_sims_: tuple of {array-like}
    simulations of in-sample residuals (for `type_pi` not conformal prediction) or
    calibrated residuals (for `type_pi` in conformal prediction)

kde_: A scikit-learn object, see https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KernelDensity.html

residuals_std_dev_: residuals standard deviation

Examples:

Example 1:

import nnetsauce as ns
import numpy as np
from sklearn import linear_model
np.random.seed(123)

M = np.random.rand(10, 3)
M[:,0] = 10*M[:,0]
M[:,2] = 25*M[:,2]
print(M)

# Adjust Bayesian Ridge
regr4 = linear_model.BayesianRidge()
obj_MTS = ns.MTS(regr4, lags = 1, n_hidden_features=5)
obj_MTS.fit(M)
print(obj_MTS.predict())

# with credible intervals
print(obj_MTS.predict(return_std=True, level=80))

print(obj_MTS.predict(return_std=True, level=95))

Example 2:

import nnetsauce as ns
import numpy as np
from sklearn import linear_model

dataset = {
'date' : ['2001-01-01', '2002-01-01', '2003-01-01', '2004-01-01', '2005-01-01'],
'series1' : [34, 30, 35.6, 33.3, 38.1],
'series2' : [4, 5.5, 5.6, 6.3, 5.1],
'series3' : [100, 100.5, 100.6, 100.2, 100.1]}
df = pd.DataFrame(dataset).set_index('date')
print(df)

# Adjust Bayesian Ridge
regr5 = linear_model.BayesianRidge()
obj_MTS = ns.MTS(regr5, lags = 1, n_hidden_features=5)
obj_MTS.fit(df)
print(obj_MTS.predict())

# with credible intervals
print(obj_MTS.predict(return_std=True, level=80))

print(obj_MTS.predict(return_std=True, level=95))
def fit(self, X, xreg=None, **kwargs):
315    def fit(self, X, xreg=None, **kwargs):
316        """Fit MTS model to training data X, with optional regressors xreg
317
318        Parameters:
319
320        X: {array-like}, shape = [n_samples, n_features]
321            Training time series, where n_samples is the number
322            of samples and n_features is the number of features;
323            X must be in increasing order (most recent observations last)
324
325        xreg: {array-like}, shape = [n_samples, n_features_xreg]
326            Additional (external) regressors to be passed to self.obj
327            xreg must be in 'increasing' order (most recent observations last)
328
329        **kwargs: for now, additional parameters to be passed to for kernel density estimation, when needed (see sklearn.neighbors.KernelDensity)
330
331        Returns:
332
333        self: object
334        """
335
336        if (
337            isinstance(X, pd.DataFrame) is False
338        ):  # input data set is a numpy array
339
340            if xreg is None:
341                X = pd.DataFrame(X)
342                self.series_names = [
343                    "series" + str(i) for i in range(X.shape[1])
344                ]
345            else:  # xreg is not None
346                X = mo.cbind(X, xreg)
347                self.xreg_ = xreg
348
349        else:  # input data set is a DataFrame with column names
350
351            X_index = None
352            if X.index is not None:
353                X_index = X.index
354            if xreg is None:
355                X = copy.deepcopy(mo.convert_df_to_numeric(X))
356            else:
357                X = copy.deepcopy(mo.cbind(mo.convert_df_to_numeric(X), xreg))
358                self.xreg_ = xreg
359            if X_index is not None:
360                X.index = X_index
361            self.series_names = X.columns.tolist()
362
363        if isinstance(X, pd.DataFrame):
364            self.df_ = X
365            X = X.values
366            self.df_.columns = self.series_names
367            self.input_dates = ts.compute_input_dates(self.df_)
368        else:
369            self.df_ = pd.DataFrame(X, columns=self.series_names)
370            self.input_dates = ts.compute_input_dates(self.df_)
371
372        try:
373            # multivariate time series
374            n, p = X.shape
375        except:
376            # univariate time series
377            n = X.shape[0]
378            p = 1
379
380        self.n_obs_ = n
381
382        rep_1_n = np.repeat(1, n)
383
384        self.y_ = None
385        self.X_ = None
386        self.n_series = p
387        self.fit_objs_.clear()
388        self.y_means_.clear()
389        residuals_ = []
390        self.residuals_ = None
391        self.residuals_sims_ = None
392        self.kde_ = None
393        self.sims_ = None
394        self.scaled_Z_ = None
395        self.centered_y_is_ = []
396
397        if p > 1:
398            # multivariate time series
399            mts_input = ts.create_train_inputs(X[::-1], self.lags)
400        else:
401            # univariate time series
402            mts_input = ts.create_train_inputs(
403                X.reshape(-1, 1)[::-1], self.lags
404            )
405
406        self.y_ = mts_input[0]
407
408        self.X_ = mts_input[1]
409
410        dummy_y, scaled_Z = self.cook_training_set(y=rep_1_n, X=self.X_)
411
412        self.scaled_Z_ = scaled_Z
413
414        # loop on all the time series and adjust self.obj.fit
415        if self.verbose > 0:
416            print(
417                f"\n Adjusting {type(self.obj).__name__} to multivariate time series... \n "
418            )
419
420        if self.show_progress is True:
421            iterator = tqdm(range(p))
422        else:
423            iterator = range(p)
424
425        if self.type_pi in (
426            "gaussian",
427            "kde",
428            "bootstrap",
429            "block-bootstrap",
430        ) or self.type_pi.startswith("vine"):
431            for i in iterator:
432                y_mean = np.mean(self.y_[:, i])
433                self.y_means_[i] = y_mean
434                centered_y_i = self.y_[:, i] - y_mean
435                self.centered_y_is_.append(centered_y_i)
436                self.obj.fit(X=scaled_Z, y=centered_y_i)
437                self.fit_objs_[i] = deepcopy(self.obj)
438                residuals_.append(
439                    (
440                        centered_y_i - self.fit_objs_[i].predict(scaled_Z)
441                    ).tolist()
442                )
443
444        if self.type_pi.startswith("scp"):
445            # split conformal prediction
446            for i in iterator:
447                n_y = self.y_.shape[0]
448                n_y_half = n_y // 2
449                first_half_idx = range(0, n_y_half)
450                second_half_idx = range(n_y_half, n_y)
451                y_mean_temp = np.mean(self.y_[first_half_idx, i])
452                centered_y_i_temp = self.y_[first_half_idx, i] - y_mean_temp
453                self.obj.fit(X=scaled_Z[first_half_idx, :], y=centered_y_i_temp)
454                # calibrated residuals actually
455                residuals_.append(
456                    (
457                        self.y_[second_half_idx, i]
458                        - (
459                            y_mean_temp
460                            + self.obj.predict(scaled_Z[second_half_idx, :])
461                        )
462                    ).tolist()
463                )
464                # fit on the second half
465                y_mean = np.mean(self.y_[second_half_idx, i])
466                self.y_means_[i] = y_mean
467                centered_y_i = self.y_[second_half_idx, i] - y_mean
468                self.obj.fit(X=scaled_Z[second_half_idx, :], y=centered_y_i)
469                self.fit_objs_[i] = deepcopy(self.obj)
470
471        self.residuals_ = np.asarray(residuals_).T
472
473        if self.type_pi == "gaussian":
474            self.gaussian_preds_std_ = np.std(self.residuals_, axis=0)
475
476        if self.type_pi.startswith("scp2"):
477            # Calculate mean and standard deviation for each column
478            data_mean = np.mean(self.residuals_, axis=0)
479            self.residuals_std_dev_ = np.std(self.residuals_, axis=0)
480            # Center and scale the array using broadcasting
481            self.residuals_ = (
482                self.residuals_ - data_mean[np.newaxis, :]
483            ) / self.residuals_std_dev_[np.newaxis, :]
484
485        if self.replications != None and "kde" in self.type_pi:
486            if self.verbose > 0:
487                print(f"\n Simulate residuals using {self.kernel} kernel... \n")
488            assert self.kernel in (
489                "gaussian",
490                "tophat",
491            ), "currently, 'kernel' must be either 'gaussian' or 'tophat'"
492            kernel_bandwidths = {"bandwidth": np.logspace(-6, 6, 150)}
493            grid = GridSearchCV(
494                KernelDensity(kernel=self.kernel, **kwargs),
495                param_grid=kernel_bandwidths,
496            )
497            grid.fit(self.residuals_)
498
499            if self.verbose > 0:
500                print(
501                    f"\n Best parameters for {self.kernel} kernel: {grid.best_params_} \n"
502                )
503
504            self.kde_ = grid.best_estimator_
505
506        return self

Fit MTS model to training data X, with optional regressors xreg

Parameters:

X: {array-like}, shape = [n_samples, n_features] Training time series, where n_samples is the number of samples and n_features is the number of features; X must be in increasing order (most recent observations last)

xreg: {array-like}, shape = [n_samples, n_features_xreg] Additional (external) regressors to be passed to self.obj xreg must be in 'increasing' order (most recent observations last)

**kwargs: for now, additional parameters to be passed to for kernel density estimation, when needed (see sklearn.neighbors.KernelDensity)

Returns:

self: object

def predict(self, h=5, level=95, **kwargs):
508    def predict(self, h=5, level=95, **kwargs):
509        """Forecast all the time series, h steps ahead
510
511        Parameters:
512
513        h: {integer}
514            Forecasting horizon
515
516        level: {integer}
517            Level of confidence (if obj has option 'return_std' and the
518            posterior is gaussian)
519
520        new_xreg: {array-like}, shape = [n_samples = h, n_new_xreg]
521            New values of additional (deterministic) regressors on horizon = h
522            new_xreg must be in increasing order (most recent observations last)
523
524        **kwargs: additional parameters to be passed to
525                self.cook_test_set
526
527        Returns:
528
529        model predictions for horizon = h: {array-like}, data frame or tuple.
530        Standard deviation and prediction intervals are returned when
531        `obj.predict` can return standard deviation
532
533        """
534
535        self.output_dates_, frequency = ts.compute_output_dates(self.df_, h)
536
537        self.level_ = level
538
539        self.return_std_ = False  # do not remove (/!\)
540
541        self.mean_ = None  # do not remove (/!\)
542
543        self.mean_ = deepcopy(self.y_)  # do not remove (/!\)
544
545        self.lower_ = None  # do not remove (/!\)
546
547        self.upper_ = None  # do not remove (/!\)
548
549        self.sims_ = None  # do not remove (/!\)
550
551        y_means_ = np.asarray([self.y_means_[i] for i in range(self.n_series)])
552
553        n_features = self.n_series * self.lags
554
555        self.alpha_ = 100 - level
556
557        pi_multiplier = norm.ppf(1 - self.alpha_ / 200)
558
559        if "return_std" in kwargs:  # bayesian forecasting
560            self.return_std_ = True
561            self.preds_std_ = []
562            DescribeResult = namedtuple(
563                "DescribeResult", ("mean", "lower", "upper")
564            )  # to be updated
565
566        if "return_pi" in kwargs:  # split conformal, without simulation
567            mean_pi_ = []
568            lower_pi_ = []
569            upper_pi_ = []
570            DescribeResult = namedtuple(
571                "DescribeResult", ("mean", "lower", "upper")
572            )  # to be updated
573
574        if self.kde_ != None and "kde" in self.type_pi:  # kde
575            if self.verbose == 1:
576                self.residuals_sims_ = tuple(
577                    self.kde_.sample(
578                        n_samples=h, random_state=self.seed + 100 * i
579                    )
580                    for i in tqdm(range(self.replications))
581                )
582            elif self.verbose == 0:
583                self.residuals_sims_ = tuple(
584                    self.kde_.sample(
585                        n_samples=h, random_state=self.seed + 100 * i
586                    )
587                    for i in range(self.replications)
588                )
589
590        if self.type_pi in ("bootstrap", "scp-bootstrap", "scp2-bootstrap"):
591            assert self.replications is not None and isinstance(
592                self.replications, int
593            ), "'replications' must be provided and be an integer"
594            if self.verbose == 1:
595                self.residuals_sims_ = tuple(
596                    ts.bootstrap(
597                        self.residuals_,
598                        h=h,
599                        block_size=None,
600                        seed=self.seed + 100 * i,
601                    )
602                    for i in tqdm(range(self.replications))
603                )
604            elif self.verbose == 0:
605                self.residuals_sims_ = tuple(
606                    ts.bootstrap(
607                        self.residuals_,
608                        h=h,
609                        block_size=None,
610                        seed=self.seed + 100 * i,
611                    )
612                    for i in range(self.replications)
613                )
614
615        if self.type_pi in (
616            "block-bootstrap",
617            "scp-block-bootstrap",
618            "scp2-block-bootstrap",
619        ):
620            if self.block_size is None:
621                self.block_size = int(
622                    np.ceil(3.15 * (self.residuals_.shape[0] ** (1 / 3)))
623                )
624
625            assert self.replications is not None and isinstance(
626                self.replications, int
627            ), "'replications' must be provided and be an integer"
628            if self.verbose == 1:
629                self.residuals_sims_ = tuple(
630                    ts.bootstrap(
631                        self.residuals_,
632                        h=h,
633                        block_size=self.block_size,
634                        seed=self.seed + 100 * i,
635                    )
636                    for i in tqdm(range(self.replications))
637                )
638            elif self.verbose == 0:
639                self.residuals_sims_ = tuple(
640                    ts.bootstrap(
641                        self.residuals_,
642                        h=h,
643                        block_size=self.block_size,
644                        seed=self.seed + 100 * i,
645                    )
646                    for i in range(self.replications)
647                )
648
649        if "vine" in self.type_pi:
650            if self.verbose == 1:
651                self.residuals_sims_ = tuple(
652                    vinecopula_sample(
653                        x=self.residuals_,
654                        n_samples=h,
655                        method=self.type_pi,
656                        random_state=self.seed + 100 * i,
657                    )
658                    for i in tqdm(range(self.replications))
659                )
660            elif self.verbose == 0:
661                self.residuals_sims_ = tuple(
662                    vinecopula_sample(
663                        x=self.residuals_,
664                        n_samples=h,
665                        method=self.type_pi,
666                        random_state=self.seed + 100 * i,
667                    )
668                    for i in range(self.replications)
669                )
670
671        for _ in range(h):
672
673            new_obs = ts.reformat_response(self.mean_, self.lags)
674
675            new_X = new_obs.reshape(1, n_features)
676
677            cooked_new_X = self.cook_test_set(new_X, **kwargs)
678
679            if "return_std" in kwargs:
680                self.preds_std_.append(
681                    [
682                        np.asarray(
683                            self.fit_objs_[i].predict(
684                                cooked_new_X, return_std=True
685                            )[1]
686                        ).item()
687                        for i in range(self.n_series)
688                    ]
689                )
690
691            if "return_pi" in kwargs:
692                for i in range(self.n_series):
693                    preds_pi = self.fit_objs_[i].predict(
694                        cooked_new_X, return_pi=True
695                    )
696                    mean_pi_.append(preds_pi.mean[0])
697                    lower_pi_.append(preds_pi.lower[0])
698                    upper_pi_.append(preds_pi.upper[0])
699
700            predicted_cooked_new_X = np.asarray(
701                [
702                    np.asarray(self.fit_objs_[i].predict(cooked_new_X)).item()
703                    for i in range(self.n_series)
704                ]
705            )
706
707            preds = np.asarray(y_means_ + predicted_cooked_new_X)
708
709            self.mean_ = mo.rbind(preds, self.mean_)  # preallocate?
710
711        # function's return ----------------------------------------------------------------------
712        self.mean_ = pd.DataFrame(
713            self.mean_[0:h, :][::-1],
714            columns=self.df_.columns,
715            index=self.output_dates_,
716        )
717
718        if (
719            (("return_std" not in kwargs) and ("return_pi" not in kwargs))
720            and (self.type_pi not in ("gaussian", "scp"))
721        ) or ("vine" in self.type_pi):
722
723            if self.replications is None:
724                return self.mean_
725
726            # if "return_std" not in kwargs and self.replications is not None
727            meanf = []
728            lower = []
729            upper = []
730
731            if "scp2" in self.type_pi:
732
733                if self.verbose == 1:
734                    self.sims_ = tuple(
735                        (
736                            self.mean_
737                            + self.residuals_sims_[i]
738                            * self.residuals_std_dev_[np.newaxis, :]
739                            for i in tqdm(range(self.replications))
740                        )
741                    )
742                elif self.verbose == 0:
743                    self.sims_ = tuple(
744                        (
745                            self.mean_
746                            + self.residuals_sims_[i]
747                            * self.residuals_std_dev_[np.newaxis, :]
748                            for i in range(self.replications)
749                        )
750                    )
751            else:
752
753                if self.verbose == 1:
754                    self.sims_ = tuple(
755                        (
756                            self.mean_ + self.residuals_sims_[i]
757                            for i in tqdm(range(self.replications))
758                        )
759                    )
760                elif self.verbose == 0:
761                    self.sims_ = tuple(
762                        (
763                            self.mean_ + self.residuals_sims_[i]
764                            for i in range(self.replications)
765                        )
766                    )
767
768            DescribeResult = namedtuple(
769                "DescribeResult", ("mean", "sims", "lower", "upper")
770            )
771            for ix in range(self.n_series):
772                sims_ix = getsims(self.sims_, ix)
773                if self.agg == "mean":
774                    meanf.append(np.mean(sims_ix, axis=1))
775                else:
776                    meanf.append(np.median(sims_ix, axis=1))
777                lower.append(np.quantile(sims_ix, q=self.alpha_ / 200, axis=1))
778                upper.append(
779                    np.quantile(sims_ix, q=1 - self.alpha_ / 200, axis=1)
780                )
781
782            self.mean_ = pd.DataFrame(
783                np.asarray(meanf).T,
784                columns=self.series_names,  # self.df_.columns,
785                index=self.output_dates_,
786            )
787
788            self.lower_ = pd.DataFrame(
789                np.asarray(lower).T,
790                columns=self.series_names,  # self.df_.columns,
791                index=self.output_dates_,
792            )
793
794            self.upper_ = pd.DataFrame(
795                np.asarray(upper).T,
796                columns=self.series_names,  # self.df_.columns,
797                index=self.output_dates_,
798            )
799
800            res = DescribeResult(
801                self.mean_, self.sims_, self.lower_, self.upper_
802            )
803
804            if self.xreg_ is not None:
805
806                if len(self.xreg_.shape) > 1:
807
808                    res2 = mx.tuple_map(
809                        res,
810                        lambda x: mo.delete_last_columns(
811                            x, num_columns=self.xreg_.shape[1]
812                        ),
813                    )
814
815                else:
816
817                    res2 = mx.tuple_map(
818                        res, lambda x: mo.delete_last_columns(x, num_columns=1)
819                    )
820
821                return res2
822
823            else:
824
825                return res
826
827        if (
828            (("return_std" in kwargs) or ("return_pi" in kwargs))
829            and (self.type_pi not in ("gaussian", "scp"))
830        ) or "vine" in self.type_pi:
831            DescribeResult = namedtuple(
832                "DescribeResult", ("mean", "lower", "upper")
833            )
834
835            self.mean_ = pd.DataFrame(
836                np.asarray(self.mean_),
837                columns=self.series_names,  # self.df_.columns,
838                index=self.output_dates_,
839            )
840
841            if "return_std" in kwargs:
842
843                self.preds_std_ = np.asarray(self.preds_std_)
844
845                self.lower_ = pd.DataFrame(
846                    self.mean_.values - pi_multiplier * self.preds_std_,
847                    columns=self.series_names,  # self.df_.columns,
848                    index=self.output_dates_,
849                )
850
851                self.upper_ = pd.DataFrame(
852                    self.mean_.values + pi_multiplier * self.preds_std_,
853                    columns=self.series_names,  # self.df_.columns,
854                    index=self.output_dates_,
855                )
856
857            if "return_pi" in kwargs:
858
859                self.lower_ = pd.DataFrame(
860                    np.asarray(lower_pi_).reshape(h, self.n_series)
861                    + y_means_[np.newaxis, :],
862                    columns=self.series_names,  # self.df_.columns,
863                    index=self.output_dates_,
864                )
865
866                self.upper_ = pd.DataFrame(
867                    np.asarray(upper_pi_).reshape(h, self.n_series)
868                    + y_means_[np.newaxis, :],
869                    columns=self.series_names,  # self.df_.columns,
870                    index=self.output_dates_,
871                )
872
873            res = DescribeResult(self.mean_, self.lower_, self.upper_)
874
875            if self.xreg_ is not None:
876                if len(self.xreg_.shape) > 1:
877                    res2 = mx.tuple_map(
878                        res,
879                        lambda x: mo.delete_last_columns(
880                            x, num_columns=self.xreg_.shape[1]
881                        ),
882                    )
883                else:
884                    res2 = mx.tuple_map(
885                        res, lambda x: mo.delete_last_columns(x, num_columns=1)
886                    )
887                return DescribeResult(res2[0], res2[1], res2[2])
888
889            return res
890
891        if self.type_pi == "gaussian":
892
893            DescribeResult = namedtuple(
894                "DescribeResult", ("mean", "lower", "upper")
895            )
896
897            self.mean_ = pd.DataFrame(
898                np.asarray(self.mean_),
899                columns=self.series_names,  # self.df_.columns,
900                index=self.output_dates_,
901            )
902
903            self.lower_ = pd.DataFrame(
904                self.mean_.values - pi_multiplier * self.gaussian_preds_std_,
905                columns=self.series_names,  # self.df_.columns,
906                index=self.output_dates_,
907            )
908
909            self.upper_ = pd.DataFrame(
910                self.mean_.values + pi_multiplier * self.gaussian_preds_std_,
911                columns=self.series_names,  # self.df_.columns,
912                index=self.output_dates_,
913            )
914
915            res = DescribeResult(self.mean_, self.lower_, self.upper_)
916
917            if self.xreg_ is not None:
918                if len(self.xreg_.shape) > 1:
919                    res2 = mx.tuple_map(
920                        res,
921                        lambda x: mo.delete_last_columns(
922                            x, num_columns=self.xreg_.shape[1]
923                        ),
924                    )
925                else:
926                    res2 = mx.tuple_map(
927                        res, lambda x: mo.delete_last_columns(x, num_columns=1)
928                    )
929                return DescribeResult(res2[0], res2[1], res2[2])
930
931            return res

Forecast all the time series, h steps ahead

Parameters:

h: {integer} Forecasting horizon

level: {integer} Level of confidence (if obj has option 'return_std' and the posterior is gaussian)

new_xreg: {array-like}, shape = [n_samples = h, n_new_xreg] New values of additional (deterministic) regressors on horizon = h new_xreg must be in increasing order (most recent observations last)

**kwargs: additional parameters to be passed to self.cook_test_set

Returns:

model predictions for horizon = h: {array-like}, data frame or tuple. Standard deviation and prediction intervals are returned when obj.predict can return standard deviation

def score(self, X, training_index, testing_index, scoring=None, **kwargs):
 933    def score(self, X, training_index, testing_index, scoring=None, **kwargs):
 934        """Train on training_index, score on testing_index."""
 935
 936        assert (
 937            bool(set(training_index).intersection(set(testing_index))) == False
 938        ), "Non-overlapping 'training_index' and 'testing_index' required"
 939
 940        # Dimensions
 941        try:
 942            # multivariate time series
 943            n, p = X.shape
 944        except:
 945            # univariate time series
 946            n = X.shape[0]
 947            p = 1
 948
 949        # Training and testing sets
 950        if p > 1:
 951            X_train = X[training_index, :]
 952            X_test = X[testing_index, :]
 953        else:
 954            X_train = X[training_index]
 955            X_test = X[testing_index]
 956
 957        # Horizon
 958        h = len(testing_index)
 959        assert (
 960            len(training_index) + h
 961        ) <= n, "Please check lengths of training and testing windows"
 962
 963        # Fit and predict
 964        self.fit(X_train, **kwargs)
 965        preds = self.predict(h=h, **kwargs)
 966
 967        if scoring is None:
 968            scoring = "neg_root_mean_squared_error"
 969
 970        # check inputs
 971        assert scoring in (
 972            "explained_variance",
 973            "neg_mean_absolute_error",
 974            "neg_mean_squared_error",
 975            "neg_root_mean_squared_error",
 976            "neg_mean_squared_log_error",
 977            "neg_median_absolute_error",
 978            "r2",
 979        ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
 980                               'neg_mean_squared_error', 'neg_root_mean_squared_error', 'neg_mean_squared_log_error', \
 981                               'neg_median_absolute_error', 'r2')"
 982
 983        scoring_options = {
 984            "explained_variance": skm2.explained_variance_score,
 985            "neg_mean_absolute_error": skm2.mean_absolute_error,
 986            "neg_mean_squared_error": skm2.mean_squared_error,
 987            "neg_root_mean_squared_error": lambda x, y: np.sqrt(
 988                skm2.mean_squared_error(x, y)
 989            ),
 990            "neg_mean_squared_log_error": skm2.mean_squared_log_error,
 991            "neg_median_absolute_error": skm2.median_absolute_error,
 992            "r2": skm2.r2_score,
 993        }
 994
 995        # if p > 1:
 996        #     return tuple(
 997        #         [
 998        #             scoring_options[scoring](
 999        #                 X_test[:, i], preds[:, i]#, **kwargs
1000        #             )
1001        #             for i in range(p)
1002        #         ]
1003        #     )
1004        # else:
1005        return scoring_options[scoring](X_test, preds)

Train on training_index, score on testing_index.

class MultitaskClassifier(nnetsauce.Base, sklearn.base.ClassifierMixin):
 16class MultitaskClassifier(Base, ClassifierMixin):
 17    """Multitask Classification model based on regression models, with shared covariates
 18
 19    Parameters:
 20
 21        obj: object
 22            any object (must be a regression model) containing a method fit (obj.fit())
 23            and a method predict (obj.predict())
 24
 25        n_hidden_features: int
 26            number of nodes in the hidden layer
 27
 28        activation_name: str
 29            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 30
 31        a: float
 32            hyperparameter for 'prelu' or 'elu' activation function
 33
 34        nodes_sim: str
 35            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 36            'uniform'
 37
 38        bias: boolean
 39            indicates if the hidden layer contains a bias term (True) or not
 40            (False)
 41
 42        dropout: float
 43            regularization parameter; (random) percentage of nodes dropped out
 44            of the training
 45
 46        direct_link: boolean
 47            indicates if the original predictors are included (True) in model's
 48            fitting or not (False)
 49
 50        n_clusters: int
 51            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 52                no clustering)
 53
 54        cluster_encode: bool
 55            defines how the variable containing clusters is treated (default is one-hot)
 56            if `False`, then labels are used, without one-hot encoding
 57
 58        type_clust: str
 59            type of clustering method: currently k-means ('kmeans') or Gaussian
 60            Mixture Model ('gmm')
 61
 62        type_scaling: a tuple of 3 strings
 63            scaling methods for inputs, hidden layer, and clustering respectively
 64            (and when relevant).
 65            Currently available: standardization ('std') or MinMax scaling ('minmax')
 66
 67        col_sample: float
 68            percentage of covariates randomly chosen for training
 69
 70        row_sample: float
 71            percentage of rows chosen for training, by stratified bootstrapping
 72
 73        seed: int
 74            reproducibility seed for nodes_sim=='uniform'
 75
 76        backend: str
 77            "cpu" or "gpu" or "tpu"
 78
 79    Attributes:
 80
 81        fit_objs_: dict
 82            objects adjusted to each individual time series
 83
 84        n_classes_: int
 85            number of classes for the classifier
 86
 87    Examples:
 88
 89    See also [https://github.com/Techtonique/nnetsauce/blob/master/examples/mtask_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/mtask_classification.py)
 90
 91    ```python
 92    import nnetsauce as ns
 93    import numpy as np
 94    from sklearn.datasets import load_breast_cancer
 95    from sklearn.linear_model import LinearRegression
 96    from sklearn.model_selection import train_test_split
 97    from sklearn import metrics
 98    from time import time
 99
100    breast_cancer = load_breast_cancer()
101    Z = breast_cancer.data
102    t = breast_cancer.target
103
104    X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2,
105                                                        random_state=123+2*10)
106
107    # Linear Regression is used
108    regr = LinearRegression()
109    fit_obj = ns.MultitaskClassifier(regr, n_hidden_features=5,
110                                n_clusters=2, type_clust="gmm")
111
112    start = time()
113    fit_obj.fit(X_train, y_train)
114    print(f"Elapsed {time() - start}")
115
116    print(fit_obj.score(X_test, y_test))
117    print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
118
119    start = time()
120    preds = fit_obj.predict(X_test)
121    print(f"Elapsed {time() - start}")
122    print(metrics.classification_report(preds, y_test))
123    ```
124
125    """
126
127    # construct the object -----
128
129    def __init__(
130        self,
131        obj,
132        n_hidden_features=5,
133        activation_name="relu",
134        a=0.01,
135        nodes_sim="sobol",
136        bias=True,
137        dropout=0,
138        direct_link=True,
139        n_clusters=2,
140        cluster_encode=True,
141        type_clust="kmeans",
142        type_scaling=("std", "std", "std"),
143        col_sample=1,
144        row_sample=1,
145        seed=123,
146        backend="cpu",
147    ):
148        super().__init__(
149            n_hidden_features=n_hidden_features,
150            activation_name=activation_name,
151            a=a,
152            nodes_sim=nodes_sim,
153            bias=bias,
154            dropout=dropout,
155            direct_link=direct_link,
156            n_clusters=n_clusters,
157            cluster_encode=cluster_encode,
158            type_clust=type_clust,
159            type_scaling=type_scaling,
160            col_sample=col_sample,
161            row_sample=row_sample,
162            seed=seed,
163            backend=backend,
164        )
165
166        self.type_fit = "classification"
167        self.obj = obj
168        self.fit_objs_ = {}
169
170    def fit(self, X, y, sample_weight=None, **kwargs):
171        """Fit MultitaskClassifier to training data (X, y).
172
173        Args:
174
175            X: {array-like}, shape = [n_samples, n_features]
176                Training vectors, where n_samples is the number
177                of samples and n_features is the number of features.
178
179            y: array-like, shape = [n_samples]
180                Target values.
181
182            **kwargs: additional parameters to be passed to
183                    self.cook_training_set or self.obj.fit
184
185        Returns:
186
187            self: object
188
189        """
190
191        assert mx.is_factor(y), "y must contain only integers"
192
193        output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
194
195        self.classes_ = np.unique(y)  # for compatibility with sklearn
196        self.n_classes_ = len(self.classes_)  # for compatibility with sklearn
197
198        # multitask response
199        Y = mo.one_hot_encode2(output_y, self.n_classes_)
200
201        # if sample_weight is None:
202        for i in range(self.n_classes_):
203            self.fit_objs_[i] = pickle.loads(
204                pickle.dumps(self.obj.fit(scaled_Z, Y[:, i], **kwargs), -1)
205            )
206
207        self.classes_ = np.unique(y)
208        return self
209
210    def predict(self, X, **kwargs):
211        """Predict test data X.
212
213        Args:
214
215            X: {array-like}, shape = [n_samples, n_features]
216                Training vectors, where n_samples is the number
217                of samples and n_features is the number of features.
218
219            **kwargs: additional parameters to be passed to
220                    self.cook_test_set
221
222        Returns:
223
224            model predictions: {array-like}
225
226        """
227
228        return np.argmax(self.predict_proba(X, **kwargs), axis=1)
229
230    def predict_proba(self, X, **kwargs):
231        """Predict probabilities for test data X.
232
233        Args:
234
235            X: {array-like}, shape = [n_samples, n_features]
236                Training vectors, where n_samples is the number
237                of samples and n_features is the number of features.
238
239            **kwargs: additional parameters to be passed to
240                    self.cook_test_set
241
242        Returns:
243
244            probability estimates for test data: {array-like}
245
246        """
247
248        shape_X = X.shape
249
250        probs = np.zeros((shape_X[0], self.n_classes_))
251
252        if len(shape_X) == 1:
253            n_features = shape_X[0]
254
255            new_X = mo.rbind(
256                X.reshape(1, n_features),
257                np.ones(n_features).reshape(1, n_features),
258            )
259
260            Z = self.cook_test_set(new_X, **kwargs)
261
262            # loop on all the classes
263            for i in range(self.n_classes_):
264                probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs)[0]
265
266        else:
267            Z = self.cook_test_set(X, **kwargs)
268
269            # loop on all the classes
270            for i in range(self.n_classes_):
271                probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs)
272
273        expit_raw_probs = expit(probs)
274
275        return expit_raw_probs / expit_raw_probs.sum(axis=1)[:, None]

Multitask Classification model based on regression models, with shared covariates

Parameters:

obj: object
    any object (must be a regression model) containing a method fit (obj.fit())
    and a method predict (obj.predict())

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original predictors are included (True) in model's
    fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

col_sample: float
    percentage of covariates randomly chosen for training

row_sample: float
    percentage of rows chosen for training, by stratified bootstrapping

seed: int
    reproducibility seed for nodes_sim=='uniform'

backend: str
    "cpu" or "gpu" or "tpu"

Attributes:

fit_objs_: dict
    objects adjusted to each individual time series

n_classes_: int
    number of classes for the classifier

Examples:

See also https://github.com/Techtonique/nnetsauce/blob/master/examples/mtask_classification.py

import nnetsauce as ns
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn import metrics
from time import time

breast_cancer = load_breast_cancer()
Z = breast_cancer.data
t = breast_cancer.target

X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2,
                                                    random_state=123+2*10)

# Linear Regression is used
regr = LinearRegression()
fit_obj = ns.MultitaskClassifier(regr, n_hidden_features=5,
                            n_clusters=2, type_clust="gmm")

start = time()
fit_obj.fit(X_train, y_train)
print(f"Elapsed {time() - start}")

print(fit_obj.score(X_test, y_test))
print(fit_obj.score(X_test, y_test, scoring="roc_auc"))

start = time()
preds = fit_obj.predict(X_test)
print(f"Elapsed {time() - start}")
print(metrics.classification_report(preds, y_test))
def fit(self, X, y, sample_weight=None, **kwargs):
170    def fit(self, X, y, sample_weight=None, **kwargs):
171        """Fit MultitaskClassifier to training data (X, y).
172
173        Args:
174
175            X: {array-like}, shape = [n_samples, n_features]
176                Training vectors, where n_samples is the number
177                of samples and n_features is the number of features.
178
179            y: array-like, shape = [n_samples]
180                Target values.
181
182            **kwargs: additional parameters to be passed to
183                    self.cook_training_set or self.obj.fit
184
185        Returns:
186
187            self: object
188
189        """
190
191        assert mx.is_factor(y), "y must contain only integers"
192
193        output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
194
195        self.classes_ = np.unique(y)  # for compatibility with sklearn
196        self.n_classes_ = len(self.classes_)  # for compatibility with sklearn
197
198        # multitask response
199        Y = mo.one_hot_encode2(output_y, self.n_classes_)
200
201        # if sample_weight is None:
202        for i in range(self.n_classes_):
203            self.fit_objs_[i] = pickle.loads(
204                pickle.dumps(self.obj.fit(scaled_Z, Y[:, i], **kwargs), -1)
205            )
206
207        self.classes_ = np.unique(y)
208        return self

Fit MultitaskClassifier to training data (X, y).

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, **kwargs):
210    def predict(self, X, **kwargs):
211        """Predict test data X.
212
213        Args:
214
215            X: {array-like}, shape = [n_samples, n_features]
216                Training vectors, where n_samples is the number
217                of samples and n_features is the number of features.
218
219            **kwargs: additional parameters to be passed to
220                    self.cook_test_set
221
222        Returns:
223
224            model predictions: {array-like}
225
226        """
227
228        return np.argmax(self.predict_proba(X, **kwargs), axis=1)

Predict test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
def predict_proba(self, X, **kwargs):
230    def predict_proba(self, X, **kwargs):
231        """Predict probabilities for test data X.
232
233        Args:
234
235            X: {array-like}, shape = [n_samples, n_features]
236                Training vectors, where n_samples is the number
237                of samples and n_features is the number of features.
238
239            **kwargs: additional parameters to be passed to
240                    self.cook_test_set
241
242        Returns:
243
244            probability estimates for test data: {array-like}
245
246        """
247
248        shape_X = X.shape
249
250        probs = np.zeros((shape_X[0], self.n_classes_))
251
252        if len(shape_X) == 1:
253            n_features = shape_X[0]
254
255            new_X = mo.rbind(
256                X.reshape(1, n_features),
257                np.ones(n_features).reshape(1, n_features),
258            )
259
260            Z = self.cook_test_set(new_X, **kwargs)
261
262            # loop on all the classes
263            for i in range(self.n_classes_):
264                probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs)[0]
265
266        else:
267            Z = self.cook_test_set(X, **kwargs)
268
269            # loop on all the classes
270            for i in range(self.n_classes_):
271                probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs)
272
273        expit_raw_probs = expit(probs)
274
275        return expit_raw_probs / expit_raw_probs.sum(axis=1)[:, None]

Predict probabilities for test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

probability estimates for test data: {array-like}
class PredictionInterval(sklearn.base.BaseEstimator, sklearn.base.RegressorMixin):
 20class PredictionInterval(BaseEstimator, RegressorMixin):
 21    """Class PredictionInterval: Obtain prediction intervals.
 22
 23    Attributes:
 24
 25        obj: an object;
 26            fitted object containing methods `fit` and `predict`
 27
 28        method: a string;
 29            method for constructing the prediction intervals.
 30            Currently "splitconformal" (default) and "localconformal"
 31
 32        level: a float;
 33            Confidence level for prediction intervals. Default is 95,
 34            equivalent to a miscoverage error of 5 (%)
 35
 36        replications: an integer;
 37            Number of replications for simulated conformal (default is `None`)
 38
 39        type_pi: a string;
 40            type of prediction interval: currently "kde" (default) or "bootstrap"
 41
 42        type_split: a string;
 43            "random" (random split of data) or "sequential" (sequential split of data)
 44
 45        seed: an integer;
 46            Reproducibility of fit (there's a random split between fitting and calibration data)
 47    """
 48
 49    def __init__(
 50        self,
 51        obj,
 52        method="splitconformal",
 53        level=95,
 54        type_pi="bootstrap",
 55        type_split="random",
 56        replications=None,
 57        kernel=None,
 58        agg="mean",
 59        seed=123,
 60    ):
 61
 62        self.obj = obj
 63        self.method = method
 64        self.level = level
 65        self.type_pi = type_pi
 66        self.type_split = type_split
 67        self.replications = replications
 68        self.kernel = kernel
 69        self.agg = agg
 70        self.seed = seed
 71        self.alpha_ = 1 - self.level / 100
 72        self.quantile_ = None
 73        self.icp_ = None
 74        self.calibrated_residuals_ = None
 75        self.scaled_calibrated_residuals_ = None
 76        self.calibrated_residuals_scaler_ = None
 77        self.kde_ = None
 78
 79    def fit(self, X, y):
 80        """Fit the `method` to training data (X, y).
 81
 82        Args:
 83
 84            X: array-like, shape = [n_samples, n_features];
 85                Training set vectors, where n_samples is the number
 86                of samples and n_features is the number of features.
 87
 88            y: array-like, shape = [n_samples, ]; Target values.
 89
 90        """
 91
 92        if self.type_split == "random":
 93            X_train, X_calibration, y_train, y_calibration = train_test_split(
 94                X, y, test_size=0.5, random_state=self.seed
 95            )
 96        elif self.type_split == "sequential":
 97            n_x = X.shape[0]
 98            n_x_half = n_x // 2
 99            first_half_idx = range(0, n_x_half)
100            second_half_idx = range(n_x_half, n_x)
101            X_train = X[first_half_idx, :]
102            X_calibration = X[second_half_idx, :]
103            y_train = y[first_half_idx]
104            y_calibration = y[second_half_idx]
105
106        if self.method == "splitconformal":
107
108            n_samples_calibration = X_calibration.shape[0]
109            self.obj.fit(X_train, y_train)
110            preds_calibration = self.obj.predict(X_calibration)
111            self.calibrated_residuals_ = y_calibration - preds_calibration
112            absolute_residuals = np.abs(self.calibrated_residuals_)
113            self.calibrated_residuals_scaler_ = StandardScaler(
114                with_mean=True, with_std=True
115            )
116            self.scaled_calibrated_residuals_ = (
117                self.calibrated_residuals_scaler_.fit_transform(
118                    self.calibrated_residuals_.reshape(-1, 1)
119                ).ravel()
120            )
121            try:
122                # numpy version >= 1.22
123                self.quantile_ = np.quantile(
124                    a=absolute_residuals, q=self.level / 100, method="higher"
125                )
126            except:
127                # numpy version < 1.22
128                self.quantile_ = np.quantile(
129                    a=absolute_residuals,
130                    q=self.level / 100,
131                    interpolation="higher",
132                )
133
134        if self.method == "localconformal":
135
136            mad_estimator = ExtraTreesRegressor()
137            normalizer = RegressorNormalizer(
138                self.obj, mad_estimator, AbsErrorErrFunc()
139            )
140            nc = RegressorNc(self.obj, AbsErrorErrFunc(), normalizer)
141            self.icp_ = IcpRegressor(nc)
142            self.icp_.fit(X_train, y_train)
143            self.icp_.calibrate(X_calibration, y_calibration)
144
145        return self
146
147    def predict(self, X, return_pi=False):
148        """Obtain predictions and prediction intervals
149
150        Args:
151
152            X: array-like, shape = [n_samples, n_features];
153                Testing set vectors, where n_samples is the number
154                of samples and n_features is the number of features.
155
156            return_pi: boolean
157                Whether the prediction interval is returned or not.
158                Default is False, for compatibility with other _estimators_.
159                If True, a tuple containing the predictions + lower and upper
160                bounds is returned.
161
162        """
163
164        pred = self.obj.predict(X)
165
166        if self.method == "splitconformal":
167
168            if self.replications is None:
169
170                if return_pi:
171
172                    DescribeResult = namedtuple(
173                        "DescribeResult", ("mean", "lower", "upper")
174                    )
175
176                    return DescribeResult(
177                        pred, pred - self.quantile_, pred + self.quantile_
178                    )
179
180                else:
181
182                    return pred
183
184            else:  # if self.replications is not None
185
186                assert self.type_pi in (
187                    "bootstrap",
188                    "kde",
189                ), "`self.type_pi` must be in ('bootstrap', 'kde')"
190
191                if self.type_pi == "bootstrap":
192                    np.random.seed(self.seed)
193                    self.residuals_sims_ = np.asarray(
194                        [
195                            np.random.choice(
196                                a=self.scaled_calibrated_residuals_,
197                                size=X.shape[0],
198                            )
199                            for _ in range(self.replications)
200                        ]
201                    ).T
202                    self.sims_ = np.asarray(
203                        [
204                            pred
205                            + self.calibrated_residuals_scaler_.scale_[0]
206                            * self.residuals_sims_[:, i].ravel()
207                            for i in range(self.replications)
208                        ]
209                    ).T
210                elif self.type_pi == "kde":
211                    self.kde_ = gaussian_kde(
212                        dataset=self.scaled_calibrated_residuals_
213                    )
214                    self.sims_ = np.asarray(
215                        [
216                            pred
217                            + self.calibrated_residuals_scaler_.scale_[0]
218                            * self.kde_.resample(
219                                size=X.shape[0], seed=self.seed + i
220                            ).ravel()
221                            for i in range(self.replications)
222                        ]
223                    ).T
224
225                self.mean_ = np.mean(self.sims_, axis=1)
226                self.lower_ = np.quantile(
227                    self.sims_, q=self.alpha_ / 200, axis=1
228                )
229                self.upper_ = np.quantile(
230                    self.sims_, q=1 - self.alpha_ / 200, axis=1
231                )
232
233                DescribeResult = namedtuple(
234                    "DescribeResult", ("mean", "sims", "lower", "upper")
235                )
236
237                return DescribeResult(
238                    self.mean_, self.sims_, self.lower_, self.upper_
239                )
240
241        if self.method == "localconformal":
242
243            if self.replications is None:
244
245                if return_pi:
246
247                    predictions_bounds = self.icp_.predict(
248                        X, significance=1 - self.level
249                    )
250                    DescribeResult = namedtuple(
251                        "DescribeResult", ("mean", "lower", "upper")
252                    )
253                    return DescribeResult(
254                        pred, predictions_bounds[:, 0], predictions_bounds[:, 1]
255                    )
256
257                else:
258
259                    return pred
260
261            else:  # if self.replications is not None
262
263                assert self.type_pi in (
264                    "bootstrap",
265                    "kde",
266                ), "`self.type_pi` must be in ('bootstrap', 'kde')"
267
268                if self.type_pi == "bootstrap":
269                    np.random.seed(self.seed)
270                    self.residuals_sims_ = np.asarray(
271                        [
272                            np.random.choice(
273                                a=self.scaled_calibrated_residuals_,
274                                size=X.shape[0],
275                            )
276                            for _ in range(self.replications)
277                        ]
278                    ).T
279                    self.sims_ = np.asarray(
280                        [
281                            pred
282                            + self.calibrated_residuals_scaler_.scale_[0]
283                            * self.residuals_sims_[:, i].ravel()
284                            for i in tqdm(range(self.replications))
285                        ]
286                    ).T
287                elif self.type_pi == "kde":
288                    self.kde_ = gaussian_kde(
289                        dataset=self.scaled_calibrated_residuals_
290                    )
291                    self.sims_ = np.asarray(
292                        [
293                            pred
294                            + self.calibrated_residuals_scaler_.scale_[0]
295                            * self.kde_.resample(
296                                size=X.shape[0], seed=self.seed + i
297                            ).ravel()
298                            for i in tqdm(range(self.replications))
299                        ]
300                    ).T
301
302                self.mean_ = np.mean(self.sims_, axis=1)
303                self.lower_ = np.quantile(
304                    self.sims_, q=self.alpha_ / 200, axis=1
305                )
306                self.upper_ = np.quantile(
307                    self.sims_, q=1 - self.alpha_ / 200, axis=1
308                )
309
310                DescribeResult = namedtuple(
311                    "DescribeResult", ("mean", "sims", "lower", "upper")
312                )
313
314                return DescribeResult(
315                    self.mean_, self.sims_, self.lower_, self.upper_
316                )

Class PredictionInterval: Obtain prediction intervals.

Attributes:

obj: an object;
    fitted object containing methods `fit` and `predict`

method: a string;
    method for constructing the prediction intervals.
    Currently "splitconformal" (default) and "localconformal"

level: a float;
    Confidence level for prediction intervals. Default is 95,
    equivalent to a miscoverage error of 5 (%)

replications: an integer;
    Number of replications for simulated conformal (default is `None`)

type_pi: a string;
    type of prediction interval: currently "kde" (default) or "bootstrap"

type_split: a string;
    "random" (random split of data) or "sequential" (sequential split of data)

seed: an integer;
    Reproducibility of fit (there's a random split between fitting and calibration data)
def fit(self, X, y):
 79    def fit(self, X, y):
 80        """Fit the `method` to training data (X, y).
 81
 82        Args:
 83
 84            X: array-like, shape = [n_samples, n_features];
 85                Training set vectors, where n_samples is the number
 86                of samples and n_features is the number of features.
 87
 88            y: array-like, shape = [n_samples, ]; Target values.
 89
 90        """
 91
 92        if self.type_split == "random":
 93            X_train, X_calibration, y_train, y_calibration = train_test_split(
 94                X, y, test_size=0.5, random_state=self.seed
 95            )
 96        elif self.type_split == "sequential":
 97            n_x = X.shape[0]
 98            n_x_half = n_x // 2
 99            first_half_idx = range(0, n_x_half)
100            second_half_idx = range(n_x_half, n_x)
101            X_train = X[first_half_idx, :]
102            X_calibration = X[second_half_idx, :]
103            y_train = y[first_half_idx]
104            y_calibration = y[second_half_idx]
105
106        if self.method == "splitconformal":
107
108            n_samples_calibration = X_calibration.shape[0]
109            self.obj.fit(X_train, y_train)
110            preds_calibration = self.obj.predict(X_calibration)
111            self.calibrated_residuals_ = y_calibration - preds_calibration
112            absolute_residuals = np.abs(self.calibrated_residuals_)
113            self.calibrated_residuals_scaler_ = StandardScaler(
114                with_mean=True, with_std=True
115            )
116            self.scaled_calibrated_residuals_ = (
117                self.calibrated_residuals_scaler_.fit_transform(
118                    self.calibrated_residuals_.reshape(-1, 1)
119                ).ravel()
120            )
121            try:
122                # numpy version >= 1.22
123                self.quantile_ = np.quantile(
124                    a=absolute_residuals, q=self.level / 100, method="higher"
125                )
126            except:
127                # numpy version < 1.22
128                self.quantile_ = np.quantile(
129                    a=absolute_residuals,
130                    q=self.level / 100,
131                    interpolation="higher",
132                )
133
134        if self.method == "localconformal":
135
136            mad_estimator = ExtraTreesRegressor()
137            normalizer = RegressorNormalizer(
138                self.obj, mad_estimator, AbsErrorErrFunc()
139            )
140            nc = RegressorNc(self.obj, AbsErrorErrFunc(), normalizer)
141            self.icp_ = IcpRegressor(nc)
142            self.icp_.fit(X_train, y_train)
143            self.icp_.calibrate(X_calibration, y_calibration)
144
145        return self

Fit the method to training data (X, y).

Args:

X: array-like, shape = [n_samples, n_features];
    Training set vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples, ]; Target values.
def predict(self, X, return_pi=False):
147    def predict(self, X, return_pi=False):
148        """Obtain predictions and prediction intervals
149
150        Args:
151
152            X: array-like, shape = [n_samples, n_features];
153                Testing set vectors, where n_samples is the number
154                of samples and n_features is the number of features.
155
156            return_pi: boolean
157                Whether the prediction interval is returned or not.
158                Default is False, for compatibility with other _estimators_.
159                If True, a tuple containing the predictions + lower and upper
160                bounds is returned.
161
162        """
163
164        pred = self.obj.predict(X)
165
166        if self.method == "splitconformal":
167
168            if self.replications is None:
169
170                if return_pi:
171
172                    DescribeResult = namedtuple(
173                        "DescribeResult", ("mean", "lower", "upper")
174                    )
175
176                    return DescribeResult(
177                        pred, pred - self.quantile_, pred + self.quantile_
178                    )
179
180                else:
181
182                    return pred
183
184            else:  # if self.replications is not None
185
186                assert self.type_pi in (
187                    "bootstrap",
188                    "kde",
189                ), "`self.type_pi` must be in ('bootstrap', 'kde')"
190
191                if self.type_pi == "bootstrap":
192                    np.random.seed(self.seed)
193                    self.residuals_sims_ = np.asarray(
194                        [
195                            np.random.choice(
196                                a=self.scaled_calibrated_residuals_,
197                                size=X.shape[0],
198                            )
199                            for _ in range(self.replications)
200                        ]
201                    ).T
202                    self.sims_ = np.asarray(
203                        [
204                            pred
205                            + self.calibrated_residuals_scaler_.scale_[0]
206                            * self.residuals_sims_[:, i].ravel()
207                            for i in range(self.replications)
208                        ]
209                    ).T
210                elif self.type_pi == "kde":
211                    self.kde_ = gaussian_kde(
212                        dataset=self.scaled_calibrated_residuals_
213                    )
214                    self.sims_ = np.asarray(
215                        [
216                            pred
217                            + self.calibrated_residuals_scaler_.scale_[0]
218                            * self.kde_.resample(
219                                size=X.shape[0], seed=self.seed + i
220                            ).ravel()
221                            for i in range(self.replications)
222                        ]
223                    ).T
224
225                self.mean_ = np.mean(self.sims_, axis=1)
226                self.lower_ = np.quantile(
227                    self.sims_, q=self.alpha_ / 200, axis=1
228                )
229                self.upper_ = np.quantile(
230                    self.sims_, q=1 - self.alpha_ / 200, axis=1
231                )
232
233                DescribeResult = namedtuple(
234                    "DescribeResult", ("mean", "sims", "lower", "upper")
235                )
236
237                return DescribeResult(
238                    self.mean_, self.sims_, self.lower_, self.upper_
239                )
240
241        if self.method == "localconformal":
242
243            if self.replications is None:
244
245                if return_pi:
246
247                    predictions_bounds = self.icp_.predict(
248                        X, significance=1 - self.level
249                    )
250                    DescribeResult = namedtuple(
251                        "DescribeResult", ("mean", "lower", "upper")
252                    )
253                    return DescribeResult(
254                        pred, predictions_bounds[:, 0], predictions_bounds[:, 1]
255                    )
256
257                else:
258
259                    return pred
260
261            else:  # if self.replications is not None
262
263                assert self.type_pi in (
264                    "bootstrap",
265                    "kde",
266                ), "`self.type_pi` must be in ('bootstrap', 'kde')"
267
268                if self.type_pi == "bootstrap":
269                    np.random.seed(self.seed)
270                    self.residuals_sims_ = np.asarray(
271                        [
272                            np.random.choice(
273                                a=self.scaled_calibrated_residuals_,
274                                size=X.shape[0],
275                            )
276                            for _ in range(self.replications)
277                        ]
278                    ).T
279                    self.sims_ = np.asarray(
280                        [
281                            pred
282                            + self.calibrated_residuals_scaler_.scale_[0]
283                            * self.residuals_sims_[:, i].ravel()
284                            for i in tqdm(range(self.replications))
285                        ]
286                    ).T
287                elif self.type_pi == "kde":
288                    self.kde_ = gaussian_kde(
289                        dataset=self.scaled_calibrated_residuals_
290                    )
291                    self.sims_ = np.asarray(
292                        [
293                            pred
294                            + self.calibrated_residuals_scaler_.scale_[0]
295                            * self.kde_.resample(
296                                size=X.shape[0], seed=self.seed + i
297                            ).ravel()
298                            for i in tqdm(range(self.replications))
299                        ]
300                    ).T
301
302                self.mean_ = np.mean(self.sims_, axis=1)
303                self.lower_ = np.quantile(
304                    self.sims_, q=self.alpha_ / 200, axis=1
305                )
306                self.upper_ = np.quantile(
307                    self.sims_, q=1 - self.alpha_ / 200, axis=1
308                )
309
310                DescribeResult = namedtuple(
311                    "DescribeResult", ("mean", "sims", "lower", "upper")
312                )
313
314                return DescribeResult(
315                    self.mean_, self.sims_, self.lower_, self.upper_
316                )

Obtain predictions and prediction intervals

Args:

X: array-like, shape = [n_samples, n_features];
    Testing set vectors, where n_samples is the number
    of samples and n_features is the number of features.

return_pi: boolean
    Whether the prediction interval is returned or not.
    Default is False, for compatibility with other _estimators_.
    If True, a tuple containing the predictions + lower and upper
    bounds is returned.
class SimpleMultitaskClassifier(nnetsauce.Base, sklearn.base.ClassifierMixin):
 17class SimpleMultitaskClassifier(Base, ClassifierMixin):
 18    """Multitask Classification model based on regression models, with shared covariates
 19
 20    Parameters:
 21
 22        obj: object
 23            any object (must be a regression model) containing a method fit (obj.fit())
 24            and a method predict (obj.predict())
 25
 26        seed: int
 27            reproducibility seed
 28
 29    Attributes:
 30
 31        fit_objs_: dict
 32            objects adjusted to each individual time series
 33
 34        n_classes_: int
 35            number of classes for the classifier
 36
 37    Examples:
 38
 39    ```python
 40    import nnetsauce as ns
 41    import numpy as np
 42    from sklearn.datasets import load_breast_cancer
 43    from sklearn.linear_model import LinearRegression
 44    from sklearn.model_selection import train_test_split
 45    from sklearn import metrics
 46    from time import time
 47
 48    breast_cancer = load_breast_cancer()
 49    Z = breast_cancer.data
 50    t = breast_cancer.target
 51
 52    X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2,
 53                                                        random_state=123+2*10)
 54
 55    # Linear Regression is used
 56    regr = LinearRegression()
 57    fit_obj = ns.SimpleMultitaskClassifier(regr)
 58
 59    start = time()
 60    fit_obj.fit(X_train, y_train)
 61    print(f"Elapsed {time() - start}")
 62
 63    print(fit_obj.score(X_test, y_test))
 64    print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
 65
 66    start = time()
 67    preds = fit_obj.predict(X_test)
 68    print(f"Elapsed {time() - start}")
 69    print(metrics.classification_report(preds, y_test))
 70    ```
 71
 72    """
 73
 74    # construct the object -----
 75
 76    def __init__(
 77        self,
 78        obj,
 79    ):
 80        self.type_fit = "classification"
 81        self.obj = obj
 82        self.fit_objs_ = {}
 83        self.X_scaler_ = StandardScaler()
 84        self.scaled_X_ = None
 85
 86    def fit(self, X, y, sample_weight=None, **kwargs):
 87        """Fit SimpleMultitaskClassifier to training data (X, y).
 88
 89        Args:
 90
 91            X: {array-like}, shape = [n_samples, n_features]
 92                Training vectors, where n_samples is the number
 93                of samples and n_features is the number of features.
 94
 95            y: array-like, shape = [n_samples]
 96                Target values.
 97
 98            **kwargs: additional parameters to be passed to
 99                    self.cook_training_set or self.obj.fit
100
101        Returns:
102
103            self: object
104
105        """
106
107        assert mx.is_factor(y), "y must contain only integers"
108
109        self.classes_ = np.unique(y)  # for compatibility with sklearn
110        self.n_classes_ = len(self.classes_)  # for compatibility with sklearn
111
112        self.scaled_X_ = self.X_scaler_.fit_transform(X)
113
114        # multitask response
115        Y = mo.one_hot_encode2(y, self.n_classes_)
116
117        # if sample_weight is None:
118        for i in range(self.n_classes_):
119            self.fit_objs_[i] = deepcopy(
120                self.obj.fit(self.scaled_X_, Y[:, i], **kwargs)
121            )
122
123        self.classes_ = np.unique(y)
124        return self
125
126    def predict(self, X, **kwargs):
127        """Predict test data X.
128
129        Args:
130
131            X: {array-like}, shape = [n_samples, n_features]
132                Training vectors, where n_samples is the number
133                of samples and n_features is the number of features.
134
135            **kwargs: additional parameters
136
137        Returns:
138
139            model predictions: {array-like}
140
141        """
142
143        return np.argmax(self.predict_proba(X, **kwargs), axis=1)
144
145    def predict_proba(self, X, **kwargs):
146        """Predict probabilities for test data X.
147
148        Args:
149
150            X: {array-like}, shape = [n_samples, n_features]
151                Training vectors, where n_samples is the number
152                of samples and n_features is the number of features.
153
154            **kwargs: additional parameters
155
156        Returns:
157
158            probability estimates for test data: {array-like}
159
160        """
161
162        shape_X = X.shape
163
164        probs = np.zeros((shape_X[0], self.n_classes_))
165
166        if len(shape_X) == 1:
167            n_features = shape_X[0]
168
169            new_X = mo.rbind(
170                X.reshape(1, n_features),
171                np.ones(n_features).reshape(1, n_features),
172            )
173
174            Z = self.X_scaler_.transform(new_X, **kwargs)
175
176            # loop on all the classes
177            for i in range(self.n_classes_):
178                probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs)[0]
179
180        else:
181            Z = self.X_scaler_.transform(X, **kwargs)
182
183            # loop on all the classes
184            for i in range(self.n_classes_):
185                probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs)
186
187        expit_raw_probs = expit(probs)
188
189        return expit_raw_probs / expit_raw_probs.sum(axis=1)[:, None]

Multitask Classification model based on regression models, with shared covariates

Parameters:

obj: object
    any object (must be a regression model) containing a method fit (obj.fit())
    and a method predict (obj.predict())

seed: int
    reproducibility seed

Attributes:

fit_objs_: dict
    objects adjusted to each individual time series

n_classes_: int
    number of classes for the classifier

Examples:

import nnetsauce as ns
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn import metrics
from time import time

breast_cancer = load_breast_cancer()
Z = breast_cancer.data
t = breast_cancer.target

X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2,
                                                    random_state=123+2*10)

# Linear Regression is used
regr = LinearRegression()
fit_obj = ns.SimpleMultitaskClassifier(regr)

start = time()
fit_obj.fit(X_train, y_train)
print(f"Elapsed {time() - start}")

print(fit_obj.score(X_test, y_test))
print(fit_obj.score(X_test, y_test, scoring="roc_auc"))

start = time()
preds = fit_obj.predict(X_test)
print(f"Elapsed {time() - start}")
print(metrics.classification_report(preds, y_test))
def fit(self, X, y, sample_weight=None, **kwargs):
 86    def fit(self, X, y, sample_weight=None, **kwargs):
 87        """Fit SimpleMultitaskClassifier to training data (X, y).
 88
 89        Args:
 90
 91            X: {array-like}, shape = [n_samples, n_features]
 92                Training vectors, where n_samples is the number
 93                of samples and n_features is the number of features.
 94
 95            y: array-like, shape = [n_samples]
 96                Target values.
 97
 98            **kwargs: additional parameters to be passed to
 99                    self.cook_training_set or self.obj.fit
100
101        Returns:
102
103            self: object
104
105        """
106
107        assert mx.is_factor(y), "y must contain only integers"
108
109        self.classes_ = np.unique(y)  # for compatibility with sklearn
110        self.n_classes_ = len(self.classes_)  # for compatibility with sklearn
111
112        self.scaled_X_ = self.X_scaler_.fit_transform(X)
113
114        # multitask response
115        Y = mo.one_hot_encode2(y, self.n_classes_)
116
117        # if sample_weight is None:
118        for i in range(self.n_classes_):
119            self.fit_objs_[i] = deepcopy(
120                self.obj.fit(self.scaled_X_, Y[:, i], **kwargs)
121            )
122
123        self.classes_ = np.unique(y)
124        return self

Fit SimpleMultitaskClassifier to training data (X, y).

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, **kwargs):
126    def predict(self, X, **kwargs):
127        """Predict test data X.
128
129        Args:
130
131            X: {array-like}, shape = [n_samples, n_features]
132                Training vectors, where n_samples is the number
133                of samples and n_features is the number of features.
134
135            **kwargs: additional parameters
136
137        Returns:
138
139            model predictions: {array-like}
140
141        """
142
143        return np.argmax(self.predict_proba(X, **kwargs), axis=1)

Predict test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters

Returns:

model predictions: {array-like}
def predict_proba(self, X, **kwargs):
145    def predict_proba(self, X, **kwargs):
146        """Predict probabilities for test data X.
147
148        Args:
149
150            X: {array-like}, shape = [n_samples, n_features]
151                Training vectors, where n_samples is the number
152                of samples and n_features is the number of features.
153
154            **kwargs: additional parameters
155
156        Returns:
157
158            probability estimates for test data: {array-like}
159
160        """
161
162        shape_X = X.shape
163
164        probs = np.zeros((shape_X[0], self.n_classes_))
165
166        if len(shape_X) == 1:
167            n_features = shape_X[0]
168
169            new_X = mo.rbind(
170                X.reshape(1, n_features),
171                np.ones(n_features).reshape(1, n_features),
172            )
173
174            Z = self.X_scaler_.transform(new_X, **kwargs)
175
176            # loop on all the classes
177            for i in range(self.n_classes_):
178                probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs)[0]
179
180        else:
181            Z = self.X_scaler_.transform(X, **kwargs)
182
183            # loop on all the classes
184            for i in range(self.n_classes_):
185                probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs)
186
187        expit_raw_probs = expit(probs)
188
189        return expit_raw_probs / expit_raw_probs.sum(axis=1)[:, None]

Predict probabilities for test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters

Returns:

probability estimates for test data: {array-like}
class Optimizer:
  9class Optimizer:
 10    """Optimizer class
 11
 12    Attributes:
 13
 14        type_optim: str
 15            type of optimizer, (currently) either 'sgd' (stochastic minibatch gradient descent)
 16            or 'scd' (stochastic minibatch coordinate descent)
 17
 18        num_iters: int
 19            number of iterations of the optimizer
 20
 21        learning_rate: float
 22            step size
 23
 24        batch_prop: float
 25            proportion of the initial data used at each optimization step
 26
 27        learning_method: str
 28            "poly" - learning rate decreasing as a polynomial function
 29            of # of iterations (default)
 30            "exp" - learning rate decreasing as an exponential function
 31            of # of iterations
 32            "momentum" - gradient descent using momentum
 33
 34        randomization: str
 35            type of randomization applied at each step
 36            "strat" - stratified subsampling (default)
 37            "shuffle" - random subsampling
 38
 39        mass: float
 40            mass on velocity, for `method` == "momentum"
 41
 42        decay: float
 43            coefficient of decrease of the learning rate for
 44            `method` == "poly" and `method` == "exp"
 45
 46        tolerance: float
 47            early stopping parameter (convergence of loss function)
 48
 49        verbose: int
 50            controls verbosity of gradient descent
 51            0 - nothing is printed
 52            1 - a progress bar is printed
 53            2 - successive loss function values are printed
 54
 55    """
 56
 57    # construct the object -----
 58
 59    def __init__(
 60        self,
 61        type_optim="sgd",
 62        num_iters=100,
 63        learning_rate=0.01,
 64        batch_prop=1.0,
 65        learning_method="momentum",
 66        randomization="strat",
 67        mass=0.9,
 68        decay=0.1,
 69        tolerance=1e-3,
 70        verbose=1,
 71    ):
 72        self.type_optim = type_optim
 73        self.num_iters = num_iters
 74        self.learning_rate = learning_rate
 75        self.batch_prop = batch_prop
 76        self.learning_method = learning_method
 77        self.randomization = randomization
 78        self.mass = mass
 79        self.decay = decay
 80        self.tolerance = tolerance
 81        self.verbose = verbose
 82        self.opt = None
 83
 84    def fit(self, loss_func, response, x0, **kwargs):
 85        """Fit GLM model to training data (X, y).
 86
 87        Args:
 88
 89            loss_func: loss function
 90
 91            response: array-like, shape = [n_samples]
 92            target variable (used for subsampling)
 93
 94            x0: array-like, shape = [n_features]
 95                initial value provided to the optimizer
 96
 97            **kwargs: additional parameters to be passed to
 98                    loss function
 99
100        Returns:
101
102            self: object
103
104        """
105
106        if self.type_optim == "scd":
107            self.results = scd(
108                loss_func,
109                response=response,
110                x=x0,
111                num_iters=self.num_iters,
112                batch_prop=self.batch_prop,
113                learning_rate=self.learning_rate,
114                learning_method=self.learning_method,
115                mass=self.mass,
116                decay=self.decay,
117                randomization=self.randomization,
118                tolerance=self.tolerance,
119                verbose=self.verbose,
120                **kwargs
121            )
122
123        if self.type_optim == "sgd":
124            self.results = sgd(
125                loss_func,
126                response=response,
127                x=x0,
128                num_iters=self.num_iters,
129                batch_prop=self.batch_prop,
130                learning_rate=self.learning_rate,
131                learning_method=self.learning_method,
132                mass=self.mass,
133                decay=self.decay,
134                randomization=self.randomization,
135                tolerance=self.tolerance,
136                verbose=self.verbose,
137                **kwargs
138            )
139
140        return self
141
142    def one_hot_encode(self, y, n_classes):
143        return one_hot_encode(y, n_classes)

Optimizer class

Attributes:

type_optim: str
    type of optimizer, (currently) either 'sgd' (stochastic minibatch gradient descent)
    or 'scd' (stochastic minibatch coordinate descent)

num_iters: int
    number of iterations of the optimizer

learning_rate: float
    step size

batch_prop: float
    proportion of the initial data used at each optimization step

learning_method: str
    "poly" - learning rate decreasing as a polynomial function
    of # of iterations (default)
    "exp" - learning rate decreasing as an exponential function
    of # of iterations
    "momentum" - gradient descent using momentum

randomization: str
    type of randomization applied at each step
    "strat" - stratified subsampling (default)
    "shuffle" - random subsampling

mass: float
    mass on velocity, for `method` == "momentum"

decay: float
    coefficient of decrease of the learning rate for
    `method` == "poly" and `method` == "exp"

tolerance: float
    early stopping parameter (convergence of loss function)

verbose: int
    controls verbosity of gradient descent
    0 - nothing is printed
    1 - a progress bar is printed
    2 - successive loss function values are printed
def fit(self, loss_func, response, x0, **kwargs):
 84    def fit(self, loss_func, response, x0, **kwargs):
 85        """Fit GLM model to training data (X, y).
 86
 87        Args:
 88
 89            loss_func: loss function
 90
 91            response: array-like, shape = [n_samples]
 92            target variable (used for subsampling)
 93
 94            x0: array-like, shape = [n_features]
 95                initial value provided to the optimizer
 96
 97            **kwargs: additional parameters to be passed to
 98                    loss function
 99
100        Returns:
101
102            self: object
103
104        """
105
106        if self.type_optim == "scd":
107            self.results = scd(
108                loss_func,
109                response=response,
110                x=x0,
111                num_iters=self.num_iters,
112                batch_prop=self.batch_prop,
113                learning_rate=self.learning_rate,
114                learning_method=self.learning_method,
115                mass=self.mass,
116                decay=self.decay,
117                randomization=self.randomization,
118                tolerance=self.tolerance,
119                verbose=self.verbose,
120                **kwargs
121            )
122
123        if self.type_optim == "sgd":
124            self.results = sgd(
125                loss_func,
126                response=response,
127                x=x0,
128                num_iters=self.num_iters,
129                batch_prop=self.batch_prop,
130                learning_rate=self.learning_rate,
131                learning_method=self.learning_method,
132                mass=self.mass,
133                decay=self.decay,
134                randomization=self.randomization,
135                tolerance=self.tolerance,
136                verbose=self.verbose,
137                **kwargs
138            )
139
140        return self

Fit GLM model to training data (X, y).

Args:

loss_func: loss function

response: array-like, shape = [n_samples]
target variable (used for subsampling)

x0: array-like, shape = [n_features]
    initial value provided to the optimizer

**kwargs: additional parameters to be passed to
        loss function

Returns:

self: object
class RandomBagRegressor(nnetsauce.randombag.bag.RandomBag, sklearn.base.RegressorMixin):
 18class RandomBagRegressor(RandomBag, RegressorMixin):
 19    """Randomized 'Bagging' Regression model
 20
 21    Parameters:
 22
 23        obj: object
 24            any object containing a method fit (obj.fit()) and a method predict
 25            (obj.predict())
 26
 27        n_estimators: int
 28            number of boosting iterations
 29
 30        n_hidden_features: int
 31            number of nodes in the hidden layer
 32
 33        activation_name: str
 34            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 35
 36        a: float
 37            hyperparameter for 'prelu' or 'elu' activation function
 38
 39        nodes_sim: str
 40            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 41            'uniform'
 42
 43        bias: boolean
 44            indicates if the hidden layer contains a bias term (True) or not
 45            (False)
 46
 47        dropout: float
 48            regularization parameter; (random) percentage of nodes dropped out
 49            of the training
 50
 51        direct_link: boolean
 52            indicates if the original predictors are included (True) in model''s
 53            fitting or not (False)
 54
 55        n_clusters: int
 56            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 57                no clustering)
 58
 59        cluster_encode: bool
 60            defines how the variable containing clusters is treated (default is one-hot)
 61            if `False`, then labels are used, without one-hot encoding
 62
 63        type_clust: str
 64            type of clustering method: currently k-means ('kmeans') or Gaussian
 65            Mixture Model ('gmm')
 66
 67        type_scaling: a tuple of 3 strings
 68            scaling methods for inputs, hidden layer, and clustering respectively
 69            (and when relevant).
 70            Currently available: standardization ('std') or MinMax scaling ('minmax')
 71
 72        col_sample: float
 73            percentage of covariates randomly chosen for training
 74
 75        row_sample: float
 76            percentage of rows chosen for training, by stratified bootstrapping
 77
 78        seed: int
 79            reproducibility seed for nodes_sim=='uniform'
 80
 81        backend: str
 82            "cpu" or "gpu" or "tpu"
 83
 84    Attributes:
 85
 86        voter_: dict
 87            dictionary containing all the fitted base-learners
 88
 89
 90    Examples:
 91
 92    ```python
 93    import numpy as np
 94    import nnetsauce as ns
 95    from sklearn.datasets import fetch_california_housing
 96    from sklearn.tree import DecisionTreeRegressor
 97    from sklearn.model_selection import train_test_split
 98
 99    X, y = fetch_california_housing(return_X_y=True, as_frame=False)
100
101    # split data into training test and test set
102    X_train, X_test, y_train, y_test = train_test_split(X, y,
103                                                        test_size=0.2, random_state=13)
104
105    # Requires further tuning
106    obj = DecisionTreeRegressor(max_depth=3, random_state=123)
107    obj2 = ns.RandomBagRegressor(obj=obj, direct_link=False,
108                                n_estimators=50,
109                                col_sample=0.9, row_sample=0.9,
110                                dropout=0, n_clusters=0, verbose=1)
111
112    obj2.fit(X_train, y_train)
113
114    print(np.sqrt(obj2.score(X_test, y_test))) # RMSE
115
116    ```
117
118    """
119
120    # construct the object -----
121
122    def __init__(
123        self,
124        obj,
125        n_estimators=10,
126        n_hidden_features=1,
127        activation_name="relu",
128        a=0.01,
129        nodes_sim="sobol",
130        bias=True,
131        dropout=0,
132        direct_link=False,
133        n_clusters=2,
134        cluster_encode=True,
135        type_clust="kmeans",
136        type_scaling=("std", "std", "std"),
137        col_sample=1,
138        row_sample=1,
139        n_jobs=None,
140        seed=123,
141        verbose=1,
142        backend="cpu",
143    ):
144        super().__init__(
145            obj=obj,
146            n_estimators=n_estimators,
147            n_hidden_features=n_hidden_features,
148            activation_name=activation_name,
149            a=a,
150            nodes_sim=nodes_sim,
151            bias=bias,
152            dropout=dropout,
153            direct_link=direct_link,
154            n_clusters=n_clusters,
155            cluster_encode=cluster_encode,
156            type_clust=type_clust,
157            type_scaling=type_scaling,
158            col_sample=col_sample,
159            row_sample=row_sample,
160            seed=seed,
161            backend=backend,
162        )
163
164        self.type_fit = "regression"
165        self.verbose = verbose
166        self.n_jobs = n_jobs
167        self.voter_ = {}
168
169    def fit(self, X, y, **kwargs):
170        """Fit Random 'Bagging' model to training data (X, y).
171
172        Args:
173
174            X: {array-like}, shape = [n_samples, n_features]
175                Training vectors, where n_samples is the number
176                of samples and n_features is the number of features.
177
178            y: array-like, shape = [n_samples]
179                Target values.
180
181            **kwargs: additional parameters to be passed to
182                    self.cook_training_set or self.obj.fit
183
184        Returns:
185
186            self: object
187
188        """
189
190        base_learner = CustomRegressor(
191            self.obj,
192            n_hidden_features=self.n_hidden_features,
193            activation_name=self.activation_name,
194            a=self.a,
195            nodes_sim=self.nodes_sim,
196            bias=self.bias,
197            dropout=self.dropout,
198            direct_link=self.direct_link,
199            n_clusters=self.n_clusters,
200            type_clust=self.type_clust,
201            type_scaling=self.type_scaling,
202            col_sample=self.col_sample,
203            row_sample=self.row_sample,
204            seed=self.seed,
205        )
206
207        # 1 - Sequential training -----
208
209        if self.n_jobs is None:
210            self.voter_ = rbagloop_regression(
211                base_learner, X, y, self.n_estimators, self.verbose, self.seed
212            )
213
214            self.n_estimators = len(self.voter_)
215
216            return self
217
218        # 2 - Parallel training -----
219        # buggy
220        # if self.n_jobs is not None:
221        def fit_estimators(m):
222            base_learner__ = pickle.loads(pickle.dumps(base_learner, -1))
223            base_learner__.set_params(seed=self.seed + m * 1000)
224            base_learner__.fit(X, y, **kwargs)
225            return base_learner__
226
227        if self.verbose == 1:
228            voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")(
229                delayed(fit_estimators)(m)
230                for m in tqdm(range(self.n_estimators))
231            )
232        else:
233            voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")(
234                delayed(fit_estimators)(m) for m in range(self.n_estimators)
235            )
236
237        self.voter_ = {i: elt for i, elt in enumerate(voters_list)}
238
239        self.n_estimators = len(self.voter_)
240
241        return self
242
243    def predict(self, X, weights=None, **kwargs):
244        """Predict for test data X.
245
246        Args:
247
248            X: {array-like}, shape = [n_samples, n_features]
249                Training vectors, where n_samples is the number
250                of samples and n_features is the number of features.
251
252            **kwargs: additional parameters to be passed to
253                    self.cook_test_set
254
255        Returns:
256
257            estimates for test data: {array-like}
258
259        """
260
261        def calculate_preds(voter, weights=None):
262            ensemble_preds = 0
263
264            n_iter = len(voter)
265
266            assert n_iter > 0, "no estimator found in `RandomBag` ensemble"
267
268            if weights is None:
269                for idx, elt in voter.items():
270                    ensemble_preds += elt.predict(X)
271
272                return ensemble_preds / n_iter
273
274            # if weights is not None:
275            for idx, elt in voter.items():
276                ensemble_preds += weights[idx] * elt.predict(X)
277
278            return ensemble_preds
279
280        # end calculate_preds ----
281
282        if weights is None:
283            return calculate_preds(self.voter_)
284
285        # if weights is not None:
286        self.weights = weights
287
288        return calculate_preds(self.voter_, weights)

Randomized 'Bagging' Regression model

Parameters:

obj: object
    any object containing a method fit (obj.fit()) and a method predict
    (obj.predict())

n_estimators: int
    number of boosting iterations

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original predictors are included (True) in model''s
    fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

col_sample: float
    percentage of covariates randomly chosen for training

row_sample: float
    percentage of rows chosen for training, by stratified bootstrapping

seed: int
    reproducibility seed for nodes_sim=='uniform'

backend: str
    "cpu" or "gpu" or "tpu"

Attributes:

voter_: dict
    dictionary containing all the fitted base-learners

Examples:

import numpy as np
import nnetsauce as ns
from sklearn.datasets import fetch_california_housing
from sklearn.tree import DecisionTreeRegressor
from sklearn.model_selection import train_test_split

X, y = fetch_california_housing(return_X_y=True, as_frame=False)

# split data into training test and test set
X_train, X_test, y_train, y_test = train_test_split(X, y,
                                                    test_size=0.2, random_state=13)

# Requires further tuning
obj = DecisionTreeRegressor(max_depth=3, random_state=123)
obj2 = ns.RandomBagRegressor(obj=obj, direct_link=False,
                            n_estimators=50,
                            col_sample=0.9, row_sample=0.9,
                            dropout=0, n_clusters=0, verbose=1)

obj2.fit(X_train, y_train)

print(np.sqrt(obj2.score(X_test, y_test))) # RMSE
def fit(self, X, y, **kwargs):
169    def fit(self, X, y, **kwargs):
170        """Fit Random 'Bagging' model to training data (X, y).
171
172        Args:
173
174            X: {array-like}, shape = [n_samples, n_features]
175                Training vectors, where n_samples is the number
176                of samples and n_features is the number of features.
177
178            y: array-like, shape = [n_samples]
179                Target values.
180
181            **kwargs: additional parameters to be passed to
182                    self.cook_training_set or self.obj.fit
183
184        Returns:
185
186            self: object
187
188        """
189
190        base_learner = CustomRegressor(
191            self.obj,
192            n_hidden_features=self.n_hidden_features,
193            activation_name=self.activation_name,
194            a=self.a,
195            nodes_sim=self.nodes_sim,
196            bias=self.bias,
197            dropout=self.dropout,
198            direct_link=self.direct_link,
199            n_clusters=self.n_clusters,
200            type_clust=self.type_clust,
201            type_scaling=self.type_scaling,
202            col_sample=self.col_sample,
203            row_sample=self.row_sample,
204            seed=self.seed,
205        )
206
207        # 1 - Sequential training -----
208
209        if self.n_jobs is None:
210            self.voter_ = rbagloop_regression(
211                base_learner, X, y, self.n_estimators, self.verbose, self.seed
212            )
213
214            self.n_estimators = len(self.voter_)
215
216            return self
217
218        # 2 - Parallel training -----
219        # buggy
220        # if self.n_jobs is not None:
221        def fit_estimators(m):
222            base_learner__ = pickle.loads(pickle.dumps(base_learner, -1))
223            base_learner__.set_params(seed=self.seed + m * 1000)
224            base_learner__.fit(X, y, **kwargs)
225            return base_learner__
226
227        if self.verbose == 1:
228            voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")(
229                delayed(fit_estimators)(m)
230                for m in tqdm(range(self.n_estimators))
231            )
232        else:
233            voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")(
234                delayed(fit_estimators)(m) for m in range(self.n_estimators)
235            )
236
237        self.voter_ = {i: elt for i, elt in enumerate(voters_list)}
238
239        self.n_estimators = len(self.voter_)
240
241        return self

Fit Random 'Bagging' model to training data (X, y).

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, weights=None, **kwargs):
243    def predict(self, X, weights=None, **kwargs):
244        """Predict for test data X.
245
246        Args:
247
248            X: {array-like}, shape = [n_samples, n_features]
249                Training vectors, where n_samples is the number
250                of samples and n_features is the number of features.
251
252            **kwargs: additional parameters to be passed to
253                    self.cook_test_set
254
255        Returns:
256
257            estimates for test data: {array-like}
258
259        """
260
261        def calculate_preds(voter, weights=None):
262            ensemble_preds = 0
263
264            n_iter = len(voter)
265
266            assert n_iter > 0, "no estimator found in `RandomBag` ensemble"
267
268            if weights is None:
269                for idx, elt in voter.items():
270                    ensemble_preds += elt.predict(X)
271
272                return ensemble_preds / n_iter
273
274            # if weights is not None:
275            for idx, elt in voter.items():
276                ensemble_preds += weights[idx] * elt.predict(X)
277
278            return ensemble_preds
279
280        # end calculate_preds ----
281
282        if weights is None:
283            return calculate_preds(self.voter_)
284
285        # if weights is not None:
286        self.weights = weights
287
288        return calculate_preds(self.voter_, weights)

Predict for test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

estimates for test data: {array-like}
class RandomBagClassifier(nnetsauce.randombag.bag.RandomBag, sklearn.base.ClassifierMixin):
 18class RandomBagClassifier(RandomBag, ClassifierMixin):
 19    """Randomized 'Bagging' Classification model
 20
 21    Parameters:
 22
 23        obj: object
 24            any object containing a method fit (obj.fit()) and a method predict
 25            (obj.predict())
 26
 27        n_estimators: int
 28            number of boosting iterations
 29
 30        n_hidden_features: int
 31            number of nodes in the hidden layer
 32
 33        activation_name: str
 34            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 35
 36        a: float
 37            hyperparameter for 'prelu' or 'elu' activation function
 38
 39        nodes_sim: str
 40            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 41            'uniform'
 42
 43        bias: boolean
 44            indicates if the hidden layer contains a bias term (True) or not
 45            (False)
 46
 47        dropout: float
 48            regularization parameter; (random) percentage of nodes dropped out
 49            of the training
 50
 51        direct_link: boolean
 52            indicates if the original predictors are included (True) in model's
 53            fitting or not (False)
 54
 55        n_clusters: int
 56            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 57                no clustering)
 58
 59        cluster_encode: bool
 60            defines how the variable containing clusters is treated (default is one-hot)
 61            if `False`, then labels are used, without one-hot encoding
 62
 63        type_clust: str
 64            type of clustering method: currently k-means ('kmeans') or Gaussian
 65            Mixture Model ('gmm')
 66
 67        type_scaling: a tuple of 3 strings
 68            scaling methods for inputs, hidden layer, and clustering respectively
 69            (and when relevant).
 70            Currently available: standardization ('std') or MinMax scaling ('minmax')
 71
 72        col_sample: float
 73            percentage of covariates randomly chosen for training
 74
 75        row_sample: float
 76            percentage of rows chosen for training, by stratified bootstrapping
 77
 78        seed: int
 79            reproducibility seed for nodes_sim=='uniform'
 80
 81        backend: str
 82            "cpu" or "gpu" or "tpu"
 83
 84    Attributes:
 85
 86        voter_: dict
 87            dictionary containing all the fitted base-learners
 88
 89
 90    Examples:
 91
 92    See also [https://github.com/Techtonique/nnetsauce/blob/master/examples/randombag_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/randombag_classification.py)
 93
 94    ```python
 95    import nnetsauce as ns
 96    from sklearn.datasets import load_breast_cancer
 97    from sklearn.tree import DecisionTreeClassifier
 98    from sklearn.model_selection import train_test_split
 99    from sklearn import metrics
100    from time import time
101
102
103    breast_cancer = load_breast_cancer()
104    Z = breast_cancer.data
105    t = breast_cancer.target
106    np.random.seed(123)
107    X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2)
108
109    # decision tree
110    clf = DecisionTreeClassifier(max_depth=2, random_state=123)
111    fit_obj = ns.RandomBagClassifier(clf, n_hidden_features=2,
112                                    direct_link=True,
113                                    n_estimators=100,
114                                    col_sample=0.9, row_sample=0.9,
115                                    dropout=0.3, n_clusters=0, verbose=1)
116
117    start = time()
118    fit_obj.fit(X_train, y_train)
119    print(f"Elapsed {time() - start}")
120
121    print(fit_obj.score(X_test, y_test))
122    print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
123
124    start = time()
125    preds = fit_obj.predict(X_test)
126    print(f"Elapsed {time() - start}")
127    print(metrics.classification_report(preds, y_test))
128    ```
129
130    """
131
132    # construct the object -----
133
134    def __init__(
135        self,
136        obj,
137        n_estimators=10,
138        n_hidden_features=1,
139        activation_name="relu",
140        a=0.01,
141        nodes_sim="sobol",
142        bias=True,
143        dropout=0,
144        direct_link=False,
145        n_clusters=2,
146        cluster_encode=True,
147        type_clust="kmeans",
148        type_scaling=("std", "std", "std"),
149        col_sample=1,
150        row_sample=1,
151        n_jobs=None,
152        seed=123,
153        verbose=1,
154        backend="cpu",
155    ):
156        super().__init__(
157            obj=obj,
158            n_estimators=n_estimators,
159            n_hidden_features=n_hidden_features,
160            activation_name=activation_name,
161            a=a,
162            nodes_sim=nodes_sim,
163            bias=bias,
164            dropout=dropout,
165            direct_link=direct_link,
166            n_clusters=n_clusters,
167            cluster_encode=cluster_encode,
168            type_clust=type_clust,
169            type_scaling=type_scaling,
170            col_sample=col_sample,
171            row_sample=row_sample,
172            seed=seed,
173            backend=backend,
174        )
175
176        self.type_fit = "classification"
177        self.verbose = verbose
178        self.n_jobs = n_jobs
179        self.voter_ = {}
180
181    def fit(self, X, y, **kwargs):
182        """Fit Random 'Bagging' model to training data (X, y).
183
184        Args:
185
186            X: {array-like}, shape = [n_samples, n_features]
187                Training vectors, where n_samples is the number
188                of samples and n_features is the number of features.
189
190            y: array-like, shape = [n_samples]
191                Target values.
192
193            **kwargs: additional parameters to be passed to
194                    self.cook_training_set or self.obj.fit
195
196        Returns:
197
198            self: object
199
200        """
201
202        assert mx.is_factor(y), "y must contain only integers"
203
204        self.n_classes_ = len(np.unique(y))  # for compatibility with sklearn
205
206        # training
207        self.n_classes = len(np.unique(y))
208
209        base_learner = CustomClassifier(
210            self.obj,
211            n_hidden_features=self.n_hidden_features,
212            activation_name=self.activation_name,
213            a=self.a,
214            nodes_sim=self.nodes_sim,
215            bias=self.bias,
216            dropout=self.dropout,
217            direct_link=self.direct_link,
218            n_clusters=self.n_clusters,
219            type_clust=self.type_clust,
220            type_scaling=self.type_scaling,
221            col_sample=self.col_sample,
222            row_sample=self.row_sample,
223            seed=self.seed,
224        )
225
226        # 1 - Sequential training -----
227
228        if self.n_jobs is None:
229            self.voter_ = rbagloop_classification(
230                base_learner, X, y, self.n_estimators, self.verbose, self.seed
231            )
232
233            self.n_estimators = len(self.voter_)
234
235            return self
236
237        # 2 - Parallel training -----
238        # buggy
239        # if self.n_jobs is not None:
240        def fit_estimators(m):
241            base_learner__ = pickle.loads(pickle.dumps(base_learner, -1))
242            base_learner__.set_params(seed=self.seed + m * 1000)
243            base_learner__.fit(X, y, **kwargs)
244            return base_learner__
245
246        if self.verbose == 1:
247            voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")(
248                delayed(fit_estimators)(m)
249                for m in tqdm(range(self.n_estimators))
250            )
251        else:
252            voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")(
253                delayed(fit_estimators)(m) for m in range(self.n_estimators)
254            )
255
256        self.voter_ = {idx: elt for idx, elt in enumerate(voters_list)}
257
258        self.n_estimators = len(self.voter_)
259        self.classes_ = np.unique(y)
260        return self
261
262    def predict(self, X, weights=None, **kwargs):
263        """Predict test data X.
264
265        Args:
266
267            X: {array-like}, shape = [n_samples, n_features]
268                Training vectors, where n_samples is the number
269                of samples and n_features is the number of features.
270
271            **kwargs: additional parameters to be passed to
272                    self.cook_test_set
273
274        Returns:
275
276            model predictions: {array-like}
277
278        """
279        return self.predict_proba(X, weights, **kwargs).argmax(axis=1)
280
281    def predict_proba(self, X, weights=None, **kwargs):
282        """Predict probabilities for test data X.
283
284        Args:
285
286            X: {array-like}, shape = [n_samples, n_features]
287                Training vectors, where n_samples is the number
288                of samples and n_features is the number of features.
289
290            **kwargs: additional parameters to be passed to
291                    self.cook_test_set
292
293        Returns:
294
295            probability estimates for test data: {array-like}
296
297        """
298
299        def calculate_probas(voter, weights=None, verbose=None):
300            ensemble_proba = 0
301
302            n_iter = len(voter)
303
304            assert n_iter > 0, "no estimator found in `RandomBag` ensemble"
305
306            if weights is None:
307                for idx, elt in voter.items():
308                    try:
309                        ensemble_proba += elt.predict_proba(X)
310
311                        # if verbose == 1:
312                        #    pbar.update(idx)
313
314                    except:
315                        continue
316
317                # if verbose == 1:
318                #    pbar.update(n_iter)
319
320                return ensemble_proba / n_iter
321
322            # if weights is not None:
323            for idx, elt in voter.items():
324                ensemble_proba += weights[idx] * elt.predict_proba(X)
325
326                # if verbose == 1:
327                #    pbar.update(idx)
328
329            # if verbose == 1:
330            #    pbar.update(n_iter)
331
332            return ensemble_proba
333
334        # end calculate_probas ----
335
336        if self.n_jobs is None:
337            # if self.verbose == 1:
338            #    pbar = Progbar(self.n_estimators)
339
340            if weights is None:
341                return calculate_probas(self.voter_, verbose=self.verbose)
342
343            # if weights is not None:
344            self.weights = weights
345
346            return calculate_probas(self.voter_, weights, verbose=self.verbose)
347
348        # if self.n_jobs is not None:
349        def predict_estimator(m):
350            try:
351                return self.voter_[m].predict_proba(X)
352            except:
353                pass
354
355        if self.verbose == 1:
356            preds = Parallel(n_jobs=self.n_jobs, prefer="threads")(
357                delayed(predict_estimator)(m)
358                for m in tqdm(range(self.n_estimators))
359            )
360
361        else:
362            preds = Parallel(n_jobs=self.n_jobs, prefer="threads")(
363                delayed(predict_estimator)(m) for m in range(self.n_estimators)
364            )
365
366        ensemble_proba = 0
367
368        if weights is None:
369            for i in range(self.n_estimators):
370                ensemble_proba += preds[i]
371
372            return ensemble_proba / self.n_estimators
373
374        for i in range(self.n_estimators):
375            ensemble_proba += weights[i] * preds[i]
376
377        return ensemble_proba

Randomized 'Bagging' Classification model

Parameters:

obj: object
    any object containing a method fit (obj.fit()) and a method predict
    (obj.predict())

n_estimators: int
    number of boosting iterations

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original predictors are included (True) in model's
    fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

col_sample: float
    percentage of covariates randomly chosen for training

row_sample: float
    percentage of rows chosen for training, by stratified bootstrapping

seed: int
    reproducibility seed for nodes_sim=='uniform'

backend: str
    "cpu" or "gpu" or "tpu"

Attributes:

voter_: dict
    dictionary containing all the fitted base-learners

Examples:

See also https://github.com/Techtonique/nnetsauce/blob/master/examples/randombag_classification.py

import nnetsauce as ns
from sklearn.datasets import load_breast_cancer
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn import metrics
from time import time


breast_cancer = load_breast_cancer()
Z = breast_cancer.data
t = breast_cancer.target
np.random.seed(123)
X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2)

# decision tree
clf = DecisionTreeClassifier(max_depth=2, random_state=123)
fit_obj = ns.RandomBagClassifier(clf, n_hidden_features=2,
                                direct_link=True,
                                n_estimators=100,
                                col_sample=0.9, row_sample=0.9,
                                dropout=0.3, n_clusters=0, verbose=1)

start = time()
fit_obj.fit(X_train, y_train)
print(f"Elapsed {time() - start}")

print(fit_obj.score(X_test, y_test))
print(fit_obj.score(X_test, y_test, scoring="roc_auc"))

start = time()
preds = fit_obj.predict(X_test)
print(f"Elapsed {time() - start}")
print(metrics.classification_report(preds, y_test))
def fit(self, X, y, **kwargs):
181    def fit(self, X, y, **kwargs):
182        """Fit Random 'Bagging' model to training data (X, y).
183
184        Args:
185
186            X: {array-like}, shape = [n_samples, n_features]
187                Training vectors, where n_samples is the number
188                of samples and n_features is the number of features.
189
190            y: array-like, shape = [n_samples]
191                Target values.
192
193            **kwargs: additional parameters to be passed to
194                    self.cook_training_set or self.obj.fit
195
196        Returns:
197
198            self: object
199
200        """
201
202        assert mx.is_factor(y), "y must contain only integers"
203
204        self.n_classes_ = len(np.unique(y))  # for compatibility with sklearn
205
206        # training
207        self.n_classes = len(np.unique(y))
208
209        base_learner = CustomClassifier(
210            self.obj,
211            n_hidden_features=self.n_hidden_features,
212            activation_name=self.activation_name,
213            a=self.a,
214            nodes_sim=self.nodes_sim,
215            bias=self.bias,
216            dropout=self.dropout,
217            direct_link=self.direct_link,
218            n_clusters=self.n_clusters,
219            type_clust=self.type_clust,
220            type_scaling=self.type_scaling,
221            col_sample=self.col_sample,
222            row_sample=self.row_sample,
223            seed=self.seed,
224        )
225
226        # 1 - Sequential training -----
227
228        if self.n_jobs is None:
229            self.voter_ = rbagloop_classification(
230                base_learner, X, y, self.n_estimators, self.verbose, self.seed
231            )
232
233            self.n_estimators = len(self.voter_)
234
235            return self
236
237        # 2 - Parallel training -----
238        # buggy
239        # if self.n_jobs is not None:
240        def fit_estimators(m):
241            base_learner__ = pickle.loads(pickle.dumps(base_learner, -1))
242            base_learner__.set_params(seed=self.seed + m * 1000)
243            base_learner__.fit(X, y, **kwargs)
244            return base_learner__
245
246        if self.verbose == 1:
247            voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")(
248                delayed(fit_estimators)(m)
249                for m in tqdm(range(self.n_estimators))
250            )
251        else:
252            voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")(
253                delayed(fit_estimators)(m) for m in range(self.n_estimators)
254            )
255
256        self.voter_ = {idx: elt for idx, elt in enumerate(voters_list)}
257
258        self.n_estimators = len(self.voter_)
259        self.classes_ = np.unique(y)
260        return self

Fit Random 'Bagging' model to training data (X, y).

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, weights=None, **kwargs):
262    def predict(self, X, weights=None, **kwargs):
263        """Predict test data X.
264
265        Args:
266
267            X: {array-like}, shape = [n_samples, n_features]
268                Training vectors, where n_samples is the number
269                of samples and n_features is the number of features.
270
271            **kwargs: additional parameters to be passed to
272                    self.cook_test_set
273
274        Returns:
275
276            model predictions: {array-like}
277
278        """
279        return self.predict_proba(X, weights, **kwargs).argmax(axis=1)

Predict test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
def predict_proba(self, X, weights=None, **kwargs):
281    def predict_proba(self, X, weights=None, **kwargs):
282        """Predict probabilities for test data X.
283
284        Args:
285
286            X: {array-like}, shape = [n_samples, n_features]
287                Training vectors, where n_samples is the number
288                of samples and n_features is the number of features.
289
290            **kwargs: additional parameters to be passed to
291                    self.cook_test_set
292
293        Returns:
294
295            probability estimates for test data: {array-like}
296
297        """
298
299        def calculate_probas(voter, weights=None, verbose=None):
300            ensemble_proba = 0
301
302            n_iter = len(voter)
303
304            assert n_iter > 0, "no estimator found in `RandomBag` ensemble"
305
306            if weights is None:
307                for idx, elt in voter.items():
308                    try:
309                        ensemble_proba += elt.predict_proba(X)
310
311                        # if verbose == 1:
312                        #    pbar.update(idx)
313
314                    except:
315                        continue
316
317                # if verbose == 1:
318                #    pbar.update(n_iter)
319
320                return ensemble_proba / n_iter
321
322            # if weights is not None:
323            for idx, elt in voter.items():
324                ensemble_proba += weights[idx] * elt.predict_proba(X)
325
326                # if verbose == 1:
327                #    pbar.update(idx)
328
329            # if verbose == 1:
330            #    pbar.update(n_iter)
331
332            return ensemble_proba
333
334        # end calculate_probas ----
335
336        if self.n_jobs is None:
337            # if self.verbose == 1:
338            #    pbar = Progbar(self.n_estimators)
339
340            if weights is None:
341                return calculate_probas(self.voter_, verbose=self.verbose)
342
343            # if weights is not None:
344            self.weights = weights
345
346            return calculate_probas(self.voter_, weights, verbose=self.verbose)
347
348        # if self.n_jobs is not None:
349        def predict_estimator(m):
350            try:
351                return self.voter_[m].predict_proba(X)
352            except:
353                pass
354
355        if self.verbose == 1:
356            preds = Parallel(n_jobs=self.n_jobs, prefer="threads")(
357                delayed(predict_estimator)(m)
358                for m in tqdm(range(self.n_estimators))
359            )
360
361        else:
362            preds = Parallel(n_jobs=self.n_jobs, prefer="threads")(
363                delayed(predict_estimator)(m) for m in range(self.n_estimators)
364            )
365
366        ensemble_proba = 0
367
368        if weights is None:
369            for i in range(self.n_estimators):
370                ensemble_proba += preds[i]
371
372            return ensemble_proba / self.n_estimators
373
374        for i in range(self.n_estimators):
375            ensemble_proba += weights[i] * preds[i]
376
377        return ensemble_proba

Predict probabilities for test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

probability estimates for test data: {array-like}
class Ridge2Regressor(nnetsauce.ridge2.ridge2.Ridge2, sklearn.base.RegressorMixin):
 21class Ridge2Regressor(Ridge2, RegressorMixin):
 22    """Ridge regression with 2 regularization parameters derived from class Ridge
 23
 24    Parameters:
 25
 26        n_hidden_features: int
 27            number of nodes in the hidden layer
 28
 29        activation_name: str
 30            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 31
 32        a: float
 33            hyperparameter for 'prelu' or 'elu' activation function
 34
 35        nodes_sim: str
 36            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 37            'uniform'
 38
 39        bias: boolean
 40            indicates if the hidden layer contains a bias term (True) or not
 41            (False)
 42
 43        dropout: float
 44            regularization parameter; (random) percentage of nodes dropped out
 45            of the training
 46
 47        n_clusters: int
 48            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 49                no clustering)
 50
 51        cluster_encode: bool
 52            defines how the variable containing clusters is treated (default is one-hot)
 53            if `False`, then labels are used, without one-hot encoding
 54
 55        type_clust: str
 56            type of clustering method: currently k-means ('kmeans') or Gaussian
 57            Mixture Model ('gmm')
 58
 59        type_scaling: a tuple of 3 strings
 60            scaling methods for inputs, hidden layer, and clustering respectively
 61            (and when relevant).
 62            Currently available: standardization ('std') or MinMax scaling ('minmax')
 63
 64        lambda1: float
 65            regularization parameter on direct link
 66
 67        lambda2: float
 68            regularization parameter on hidden layer
 69
 70        seed: int
 71            reproducibility seed for nodes_sim=='uniform'
 72
 73        backend: str
 74            'cpu' or 'gpu' or 'tpu'
 75
 76    Attributes:
 77
 78        beta_: {array-like}
 79            regression coefficients
 80
 81        y_mean_: float
 82            average response
 83
 84    """
 85
 86    # construct the object -----
 87
 88    def __init__(
 89        self,
 90        n_hidden_features=5,
 91        activation_name="relu",
 92        a=0.01,
 93        nodes_sim="sobol",
 94        bias=True,
 95        dropout=0,
 96        n_clusters=2,
 97        cluster_encode=True,
 98        type_clust="kmeans",
 99        type_scaling=("std", "std", "std"),
100        lambda1=0.1,
101        lambda2=0.1,
102        seed=123,
103        backend="cpu",
104    ):
105        super().__init__(
106            n_hidden_features=n_hidden_features,
107            activation_name=activation_name,
108            a=a,
109            nodes_sim=nodes_sim,
110            bias=bias,
111            dropout=dropout,
112            n_clusters=n_clusters,
113            cluster_encode=cluster_encode,
114            type_clust=type_clust,
115            type_scaling=type_scaling,
116            lambda1=lambda1,
117            lambda2=lambda2,
118            seed=seed,
119            backend=backend,
120        )
121
122        self.type_fit = "regression"
123
124    def fit(self, X, y, **kwargs):
125        """Fit Ridge model to training data (X, y).
126
127        Args:
128
129            X: {array-like}, shape = [n_samples, n_features]
130                Training vectors, where n_samples is the number
131                of samples and n_features is the number of features.
132
133            y: array-like, shape = [n_samples]
134                Target values.
135
136            **kwargs: additional parameters to be passed to
137                    self.cook_training_set or self.obj.fit
138
139        Returns:
140
141            self: object
142
143        """
144
145        sys_platform = platform.system()
146
147        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
148
149        n_X, p_X = X.shape
150        n_Z, p_Z = scaled_Z.shape
151
152        if self.n_clusters > 0:
153            if self.encode_clusters == True:
154                n_features = p_X + self.n_clusters
155            else:
156                n_features = p_X + 1
157        else:
158            n_features = p_X
159
160        X_ = scaled_Z[:, 0:n_features]
161        Phi_X_ = scaled_Z[:, n_features:p_Z]
162
163        B = mo.crossprod(x=X_, backend=self.backend) + self.lambda1 * np.diag(
164            np.repeat(1, n_features)
165        )
166        C = mo.crossprod(x=Phi_X_, y=X_, backend=self.backend)
167        D = mo.crossprod(
168            x=Phi_X_, backend=self.backend
169        ) + self.lambda2 * np.diag(np.repeat(1, Phi_X_.shape[1]))
170
171        if sys_platform in ("Linux", "Darwin"):
172            B_inv = pinv(B) if self.backend == "cpu" else jpinv(B)
173        else:
174            B_inv = pinv(B)
175
176        W = mo.safe_sparse_dot(a=C, b=B_inv, backend=self.backend)
177        S_mat = D - mo.tcrossprod(x=W, y=C, backend=self.backend)
178
179        if sys_platform in ("Linux", "Darwin"):
180            S_inv = pinv(S_mat) if self.backend == "cpu" else jpinv(S_mat)
181        else:
182            S_inv = pinv(S_mat)
183
184        Y = mo.safe_sparse_dot(a=S_inv, b=W, backend=self.backend)
185        inv = mo.rbind(
186            mo.cbind(
187                x=B_inv + mo.crossprod(x=W, y=Y, backend=self.backend),
188                y=-np.transpose(Y),
189                backend=self.backend,
190            ),
191            mo.cbind(x=-Y, y=S_inv, backend=self.backend),
192            backend=self.backend,
193        )
194
195        self.beta_ = mo.safe_sparse_dot(
196            a=inv,
197            b=mo.crossprod(x=scaled_Z, y=centered_y, backend=self.backend),
198            backend=self.backend,
199        )
200
201        return self
202
203    def predict(self, X, **kwargs):
204        """Predict test data X.
205
206        Args:
207
208            X: {array-like}, shape = [n_samples, n_features]
209                Training vectors, where n_samples is the number
210                of samples and n_features is the number of features.
211
212            **kwargs: additional parameters to be passed to
213                    self.cook_test_set
214
215        Returns:
216
217            model predictions: {array-like}
218
219        """
220
221        if len(X.shape) == 1:
222            n_features = X.shape[0]
223            new_X = mo.rbind(
224                x=X.reshape(1, n_features),
225                y=np.ones(n_features).reshape(1, n_features),
226                backend=self.backend,
227            )
228
229            return (
230                self.y_mean_
231                + mo.safe_sparse_dot(
232                    a=self.cook_test_set(new_X, **kwargs),
233                    b=self.beta_,
234                    backend=self.backend,
235                )
236            )[0]
237
238        return self.y_mean_ + mo.safe_sparse_dot(
239            a=self.cook_test_set(X, **kwargs),
240            b=self.beta_,
241            backend=self.backend,
242        )

Ridge regression with 2 regularization parameters derived from class Ridge

Parameters:

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

lambda1: float
    regularization parameter on direct link

lambda2: float
    regularization parameter on hidden layer

seed: int
    reproducibility seed for nodes_sim=='uniform'

backend: str
    'cpu' or 'gpu' or 'tpu'

Attributes:

beta_: {array-like}
    regression coefficients

y_mean_: float
    average response
def fit(self, X, y, **kwargs):
124    def fit(self, X, y, **kwargs):
125        """Fit Ridge model to training data (X, y).
126
127        Args:
128
129            X: {array-like}, shape = [n_samples, n_features]
130                Training vectors, where n_samples is the number
131                of samples and n_features is the number of features.
132
133            y: array-like, shape = [n_samples]
134                Target values.
135
136            **kwargs: additional parameters to be passed to
137                    self.cook_training_set or self.obj.fit
138
139        Returns:
140
141            self: object
142
143        """
144
145        sys_platform = platform.system()
146
147        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
148
149        n_X, p_X = X.shape
150        n_Z, p_Z = scaled_Z.shape
151
152        if self.n_clusters > 0:
153            if self.encode_clusters == True:
154                n_features = p_X + self.n_clusters
155            else:
156                n_features = p_X + 1
157        else:
158            n_features = p_X
159
160        X_ = scaled_Z[:, 0:n_features]
161        Phi_X_ = scaled_Z[:, n_features:p_Z]
162
163        B = mo.crossprod(x=X_, backend=self.backend) + self.lambda1 * np.diag(
164            np.repeat(1, n_features)
165        )
166        C = mo.crossprod(x=Phi_X_, y=X_, backend=self.backend)
167        D = mo.crossprod(
168            x=Phi_X_, backend=self.backend
169        ) + self.lambda2 * np.diag(np.repeat(1, Phi_X_.shape[1]))
170
171        if sys_platform in ("Linux", "Darwin"):
172            B_inv = pinv(B) if self.backend == "cpu" else jpinv(B)
173        else:
174            B_inv = pinv(B)
175
176        W = mo.safe_sparse_dot(a=C, b=B_inv, backend=self.backend)
177        S_mat = D - mo.tcrossprod(x=W, y=C, backend=self.backend)
178
179        if sys_platform in ("Linux", "Darwin"):
180            S_inv = pinv(S_mat) if self.backend == "cpu" else jpinv(S_mat)
181        else:
182            S_inv = pinv(S_mat)
183
184        Y = mo.safe_sparse_dot(a=S_inv, b=W, backend=self.backend)
185        inv = mo.rbind(
186            mo.cbind(
187                x=B_inv + mo.crossprod(x=W, y=Y, backend=self.backend),
188                y=-np.transpose(Y),
189                backend=self.backend,
190            ),
191            mo.cbind(x=-Y, y=S_inv, backend=self.backend),
192            backend=self.backend,
193        )
194
195        self.beta_ = mo.safe_sparse_dot(
196            a=inv,
197            b=mo.crossprod(x=scaled_Z, y=centered_y, backend=self.backend),
198            backend=self.backend,
199        )
200
201        return self

Fit Ridge model to training data (X, y).

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, **kwargs):
203    def predict(self, X, **kwargs):
204        """Predict test data X.
205
206        Args:
207
208            X: {array-like}, shape = [n_samples, n_features]
209                Training vectors, where n_samples is the number
210                of samples and n_features is the number of features.
211
212            **kwargs: additional parameters to be passed to
213                    self.cook_test_set
214
215        Returns:
216
217            model predictions: {array-like}
218
219        """
220
221        if len(X.shape) == 1:
222            n_features = X.shape[0]
223            new_X = mo.rbind(
224                x=X.reshape(1, n_features),
225                y=np.ones(n_features).reshape(1, n_features),
226                backend=self.backend,
227            )
228
229            return (
230                self.y_mean_
231                + mo.safe_sparse_dot(
232                    a=self.cook_test_set(new_X, **kwargs),
233                    b=self.beta_,
234                    backend=self.backend,
235                )
236            )[0]
237
238        return self.y_mean_ + mo.safe_sparse_dot(
239            a=self.cook_test_set(X, **kwargs),
240            b=self.beta_,
241            backend=self.backend,
242        )

Predict test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
class Ridge2Classifier(nnetsauce.ridge2.ridge2.Ridge2, sklearn.base.ClassifierMixin):
 18class Ridge2Classifier(Ridge2, ClassifierMixin):
 19    """Multinomial logit classification with 2 regularization parameters
 20
 21    Parameters:
 22
 23        n_hidden_features: int
 24            number of nodes in the hidden layer
 25
 26        activation_name: str
 27            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 28
 29        a: float
 30            hyperparameter for 'prelu' or 'elu' activation function
 31
 32        nodes_sim: str
 33            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 34            'uniform'
 35
 36        bias: boolean
 37            indicates if the hidden layer contains a bias term (True) or not
 38            (False)
 39
 40        dropout: float
 41            regularization parameter; (random) percentage of nodes dropped out
 42            of the training
 43
 44        direct_link: boolean
 45            indicates if the original predictors are included (True) in model's
 46            fitting or not (False)
 47
 48        n_clusters: int
 49            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 50                no clustering)
 51
 52        cluster_encode: bool
 53            defines how the variable containing clusters is treated (default is one-hot)
 54            if `False`, then labels are used, without one-hot encoding
 55
 56        type_clust: str
 57            type of clustering method: currently k-means ('kmeans') or Gaussian
 58            Mixture Model ('gmm')
 59
 60        type_scaling: a tuple of 3 strings
 61            scaling methods for inputs, hidden layer, and clustering respectively
 62            (and when relevant).
 63            Currently available: standardization ('std') or MinMax scaling ('minmax')
 64
 65        lambda1: float
 66            regularization parameter on direct link
 67
 68        lambda2: float
 69            regularization parameter on hidden layer
 70
 71        solver: str
 72            optimization function "L-BFGS-B",  "Newton-CG",
 73            "trust-ncg", "L-BFGS-B-lstsq", "Newton-CG-lstsq",
 74            "trust-ncg-lstsq" (see scipy.optimize.minimize)
 75            When using "L-BFGS-B-lstsq", "Newton-CG-lstsq", or "trust-ncg-lstsq",
 76            the initial value for the optimization is set to the least squares solution
 77
 78        seed: int
 79            reproducibility seed for nodes_sim=='uniform'
 80
 81        backend: str
 82            "cpu" or "gpu" or "tpu"
 83
 84    Attributes:
 85
 86        beta_: {array-like}
 87            regression coefficients
 88
 89        classes_: {array-like}
 90            unique classes in the target variable
 91
 92        minloglik_: float
 93            minimum value of the negative log-likelihood
 94
 95    Examples:
 96
 97    See also [https://github.com/Techtonique/nnetsauce/blob/master/examples/ridge_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/ridge_classification.py)
 98
 99    ```python
100    import nnetsauce as ns
101    import numpy as np
102    from sklearn.datasets import load_breast_cancer
103    from sklearn.model_selection import train_test_split
104    from time import time
105
106
107    breast_cancer = load_breast_cancer()
108    X = breast_cancer.data
109    y = breast_cancer.target
110
111    # split data into training test and test set
112    np.random.seed(123)
113    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
114
115    # create the model with nnetsauce
116    fit_obj = ns.Ridge2Classifier(lambda1 = 6.90185578e+04,
117                                lambda2 = 3.17392781e+02,
118                                n_hidden_features=95,
119                                n_clusters=2,
120                                dropout = 3.62817383e-01,
121                                type_clust = "gmm")
122
123    # fit the model on training set
124    start = time()
125    fit_obj.fit(X_train, y_train)
126    print(f"Elapsed {time() - start}")
127
128    # get the accuracy on test set
129    start = time()
130    print(fit_obj.score(X_test, y_test))
131    print(f"Elapsed {time() - start}")
132
133    # get area under the curve on test set (auc)
134    print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
135    ```
136
137
138    """
139
140    # construct the object -----
141
142    def __init__(
143        self,
144        n_hidden_features=5,
145        activation_name="relu",
146        a=0.01,
147        nodes_sim="sobol",
148        bias=True,
149        dropout=0,
150        direct_link=True,
151        n_clusters=2,
152        cluster_encode=True,
153        type_clust="kmeans",
154        type_scaling=("std", "std", "std"),
155        lambda1=0.1,
156        lambda2=0.1,
157        solver="L-BFGS-B",
158        seed=123,
159        backend="cpu",
160    ):
161        super().__init__(
162            n_hidden_features=n_hidden_features,
163            activation_name=activation_name,
164            a=a,
165            nodes_sim=nodes_sim,
166            bias=bias,
167            dropout=dropout,
168            direct_link=direct_link,
169            n_clusters=n_clusters,
170            cluster_encode=cluster_encode,
171            type_clust=type_clust,
172            type_scaling=type_scaling,
173            lambda1=lambda1,
174            lambda2=lambda2,
175            seed=seed,
176            backend=backend,
177        )
178
179        self.type_fit = "classification"
180        self.solver = solver
181        self.beta_ = None
182        self.classes_ = None
183        self.minloglik_ = None
184
185    def loglik(self, X, Y, **kwargs):
186        """Log-likelihood for training data (X, Y).
187
188        Args:
189
190            X: {array-like}, shape = [n_samples, n_features]
191                Training vectors, where n_samples is the number
192                of samples and n_features is the number of features.
193
194            Y: array-like, shape = [n_samples]
195                One-hot encode target values.
196
197            **kwargs: additional parameters to be passed to
198                    self.cook_training_set or self.obj.fit
199
200        Returns:
201
202        """
203
204        def loglik_grad_hess(Y, X, B, XB, hessian=True, **kwargs):
205            # nobs, n_classes
206            n, K = Y.shape
207
208            # total number of covariates
209            p = X.shape[1]
210
211            # initial number of covariates
212            init_p = p - self.n_hidden_features
213
214            max_double = 709.0
215            XB[XB > max_double] = max_double
216            exp_XB = np.exp(XB)
217            probs = exp_XB / exp_XB.sum(axis=1)[:, None]
218
219            # gradient -----
220            # (Y - p) -> (n, K)
221            # X -> (n, p)
222            # (K, n) %*% (n, p) -> (K, p)
223            if hessian is False:
224                grad = (
225                    -mo.safe_sparse_dot(
226                        a=(Y - probs).T, b=X, backend=self.backend
227                    )
228                    / n
229                )
230                grad += self.lambda1 * B[0:init_p, :].sum(axis=0)[:, None]
231                grad += self.lambda2 * B[init_p:p, :].sum(axis=0)[:, None]
232
233                return grad.flatten()
234
235            # hessian -----
236            if hessian is True:
237                Kp = K * p
238                hess = np.zeros((Kp, Kp), float)
239                for k1 in range(K):
240                    x_index = range(k1 * p, (k1 + 1) * p)
241                    for k2 in range(k1, K):
242                        y_index = range(k2 * p, (k2 + 1) * p)
243                        H_sub = (
244                            -mo.safe_sparse_dot(
245                                a=X.T,
246                                b=(probs[:, k1] * probs[:, k2])[:, None] * X,
247                                backend=self.backend,
248                            )
249                            / n
250                        )  # do not store
251                        hess[np.ix_(x_index, y_index)] = hess[
252                            np.ix_(y_index, x_index)
253                        ] = H_sub
254
255                return hess + (self.lambda1 + self.lambda2) * np.identity(Kp)
256
257        # total number of covariates
258        p = X.shape[1]
259
260        # initial number of covariates
261        init_p = p - self.n_hidden_features
262
263        # log-likelihood (1st return)
264        def loglik_func(x):
265            # (p, K)
266            B = x.reshape(Y.shape[1], p).T
267
268            # (n, K)
269            XB = mo.safe_sparse_dot(X, B, backend=self.backend)
270
271            res = -(np.sum(Y * XB, axis=1) - logsumexp(XB)).mean()
272
273            res += (
274                0.5
275                * self.lambda1
276                * mo.squared_norm(B[0:init_p, :], backend=self.backend)
277            )
278            res += (
279                0.5
280                * self.lambda2
281                * mo.squared_norm(B[init_p:p, :], backend=self.backend)
282            )
283
284            return res
285
286        # gradient of log-likelihood
287        def grad_func(x):
288            # (p, K)
289            B = x.reshape(Y.shape[1], p).T
290
291            return loglik_grad_hess(
292                Y=Y,
293                X=X,
294                B=B,
295                XB=mo.safe_sparse_dot(X, B, backend=self.backend),
296                hessian=False,
297                **kwargs
298            )
299
300        # hessian of log-likelihood
301        def hessian_func(x):
302            # (p, K)
303            B = x.reshape(Y.shape[1], p).T
304
305            return loglik_grad_hess(
306                Y=Y,
307                X=X,
308                B=B,
309                XB=mo.safe_sparse_dot(X, B, backend=self.backend),
310                hessian=True,
311                **kwargs
312            )
313
314        return loglik_func, grad_func, hessian_func
315
316    # newton-cg
317    # L-BFGS-B
318    def fit(self, X, y, **kwargs):
319        """Fit Ridge model to training data (X, y).
320
321        for beta: regression coeffs (beta11, ..., beta1p, ..., betaK1, ..., betaKp)
322        for K classes and p covariates.
323
324        Args:
325
326            X: {array-like}, shape = [n_samples, n_features]
327                Training vectors, where n_samples is the number
328                of samples and n_features is the number of features.
329
330            y: array-like, shape = [n_samples]
331                Target values.
332
333            **kwargs: additional parameters to be passed to
334                    self.cook_training_set or self.obj.fit
335
336        Returns:
337
338            self: object
339
340        """
341
342        assert mx.is_factor(y), "y must contain only integers"
343
344        output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
345
346        self.n_classes = len(np.unique(y))
347        self.classes_ = np.unique(y)  # for compatibility with sklearn
348        self.n_classes_ = len(self.classes_)  # for compatibility with sklearn
349
350        Y = mo.one_hot_encode2(output_y, self.n_classes)
351
352        # optimize for beta, minimize self.loglik (maximize loglik) -----
353        loglik_func, grad_func, hessian_func = self.loglik(X=scaled_Z, Y=Y)
354
355        if self.solver == "L-BFGS-B":
356            opt = minimize(
357                fun=loglik_func,
358                x0=np.zeros(scaled_Z.shape[1] * self.n_classes),
359                jac=grad_func,
360                method=self.solver,
361            )
362            self.beta_ = opt.x
363            self.minloglik_ = opt.fun
364
365        if self.solver in ("Newton-CG", "trust-ncg"):
366            opt = minimize(
367                fun=loglik_func,
368                x0=np.zeros(scaled_Z.shape[1] * self.n_classes),
369                jac=grad_func,
370                hess=hessian_func,
371                method=self.solver,
372            )
373            self.beta_ = opt.x
374            self.minloglik_ = opt.fun
375
376        if self.solver == "L-BFGS-B-lstsq":
377            opt = minimize(
378                fun=loglik_func,
379                x0=np.linalg.lstsq(scaled_Z, Y, rcond=None)[0].flatten(
380                    order="F"
381                ),
382                jac=grad_func,
383                method="L-BFGS-B",
384            )
385            self.beta_ = opt.x
386            self.minloglik_ = opt.fun
387
388        if self.solver in "Newton-CG-lstsq":
389            opt = minimize(
390                fun=loglik_func,
391                x0=np.linalg.lstsq(scaled_Z, Y, rcond=None)[0].flatten(
392                    order="F"
393                ),
394                jac=grad_func,
395                hess=hessian_func,
396                method="Newton-CG",
397            )
398            self.beta_ = opt.x
399            self.minloglik_ = opt.fun
400
401        if self.solver in "trust-ncg-lstsq":
402            opt = minimize(
403                fun=loglik_func,
404                x0=np.linalg.lstsq(scaled_Z, Y, rcond=None)[0].flatten(
405                    order="F"
406                ),
407                jac=grad_func,
408                hess=hessian_func,
409                method="trust-ncg",
410            )
411            self.beta_ = opt.x
412            self.minloglik_ = opt.fun
413
414        self.classes_ = np.unique(y)
415
416        return self
417
418    def predict(self, X, **kwargs):
419        """Predict test data X.
420
421        Args:
422
423            X: {array-like}, shape = [n_samples, n_features]
424                Training vectors, where n_samples is the number
425                of samples and n_features is the number of features.
426
427            **kwargs: additional parameters to be passed to
428                    self.cook_test_set
429
430        Returns:
431
432            model predictions: {array-like}
433        """
434
435        return np.argmax(self.predict_proba(X, **kwargs), axis=1)
436
437    def predict_proba(self, X, **kwargs):
438        """Predict probabilities for test data X.
439
440        Args:
441
442            X: {array-like}, shape = [n_samples, n_features]
443                Training vectors, where n_samples is the number
444                of samples and n_features is the number of features.
445
446            **kwargs: additional parameters to be passed to
447                    self.cook_test_set
448
449        Returns:
450
451            probability estimates for test data: {array-like}
452
453        """
454        if len(X.shape) == 1:
455            n_features = X.shape[0]
456            new_X = mo.rbind(
457                X.reshape(1, n_features),
458                np.ones(n_features).reshape(1, n_features),
459            )
460
461            Z = self.cook_test_set(new_X, **kwargs)
462
463        else:
464            Z = self.cook_test_set(X, **kwargs)
465
466        ZB = mo.safe_sparse_dot(
467            a=Z,
468            b=self.beta_.reshape(
469                self.n_classes,
470                X.shape[1] + self.n_hidden_features + self.n_clusters,
471            ).T,
472            backend=self.backend,
473        )
474
475        exp_ZB = np.exp(ZB)
476
477        return exp_ZB / exp_ZB.sum(axis=1)[:, None]

Multinomial logit classification with 2 regularization parameters

Parameters:

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original predictors are included (True) in model's
    fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

lambda1: float
    regularization parameter on direct link

lambda2: float
    regularization parameter on hidden layer

solver: str
    optimization function "L-BFGS-B",  "Newton-CG",
    "trust-ncg", "L-BFGS-B-lstsq", "Newton-CG-lstsq",
    "trust-ncg-lstsq" (see scipy.optimize.minimize)
    When using "L-BFGS-B-lstsq", "Newton-CG-lstsq", or "trust-ncg-lstsq",
    the initial value for the optimization is set to the least squares solution

seed: int
    reproducibility seed for nodes_sim=='uniform'

backend: str
    "cpu" or "gpu" or "tpu"

Attributes:

beta_: {array-like}
    regression coefficients

classes_: {array-like}
    unique classes in the target variable

minloglik_: float
    minimum value of the negative log-likelihood

Examples:

See also https://github.com/Techtonique/nnetsauce/blob/master/examples/ridge_classification.py

import nnetsauce as ns
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from time import time


breast_cancer = load_breast_cancer()
X = breast_cancer.data
y = breast_cancer.target

# split data into training test and test set
np.random.seed(123)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

# create the model with nnetsauce
fit_obj = ns.Ridge2Classifier(lambda1 = 6.90185578e+04,
                            lambda2 = 3.17392781e+02,
                            n_hidden_features=95,
                            n_clusters=2,
                            dropout = 3.62817383e-01,
                            type_clust = "gmm")

# fit the model on training set
start = time()
fit_obj.fit(X_train, y_train)
print(f"Elapsed {time() - start}")

# get the accuracy on test set
start = time()
print(fit_obj.score(X_test, y_test))
print(f"Elapsed {time() - start}")

# get area under the curve on test set (auc)
print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
def fit(self, X, y, **kwargs):
318    def fit(self, X, y, **kwargs):
319        """Fit Ridge model to training data (X, y).
320
321        for beta: regression coeffs (beta11, ..., beta1p, ..., betaK1, ..., betaKp)
322        for K classes and p covariates.
323
324        Args:
325
326            X: {array-like}, shape = [n_samples, n_features]
327                Training vectors, where n_samples is the number
328                of samples and n_features is the number of features.
329
330            y: array-like, shape = [n_samples]
331                Target values.
332
333            **kwargs: additional parameters to be passed to
334                    self.cook_training_set or self.obj.fit
335
336        Returns:
337
338            self: object
339
340        """
341
342        assert mx.is_factor(y), "y must contain only integers"
343
344        output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
345
346        self.n_classes = len(np.unique(y))
347        self.classes_ = np.unique(y)  # for compatibility with sklearn
348        self.n_classes_ = len(self.classes_)  # for compatibility with sklearn
349
350        Y = mo.one_hot_encode2(output_y, self.n_classes)
351
352        # optimize for beta, minimize self.loglik (maximize loglik) -----
353        loglik_func, grad_func, hessian_func = self.loglik(X=scaled_Z, Y=Y)
354
355        if self.solver == "L-BFGS-B":
356            opt = minimize(
357                fun=loglik_func,
358                x0=np.zeros(scaled_Z.shape[1] * self.n_classes),
359                jac=grad_func,
360                method=self.solver,
361            )
362            self.beta_ = opt.x
363            self.minloglik_ = opt.fun
364
365        if self.solver in ("Newton-CG", "trust-ncg"):
366            opt = minimize(
367                fun=loglik_func,
368                x0=np.zeros(scaled_Z.shape[1] * self.n_classes),
369                jac=grad_func,
370                hess=hessian_func,
371                method=self.solver,
372            )
373            self.beta_ = opt.x
374            self.minloglik_ = opt.fun
375
376        if self.solver == "L-BFGS-B-lstsq":
377            opt = minimize(
378                fun=loglik_func,
379                x0=np.linalg.lstsq(scaled_Z, Y, rcond=None)[0].flatten(
380                    order="F"
381                ),
382                jac=grad_func,
383                method="L-BFGS-B",
384            )
385            self.beta_ = opt.x
386            self.minloglik_ = opt.fun
387
388        if self.solver in "Newton-CG-lstsq":
389            opt = minimize(
390                fun=loglik_func,
391                x0=np.linalg.lstsq(scaled_Z, Y, rcond=None)[0].flatten(
392                    order="F"
393                ),
394                jac=grad_func,
395                hess=hessian_func,
396                method="Newton-CG",
397            )
398            self.beta_ = opt.x
399            self.minloglik_ = opt.fun
400
401        if self.solver in "trust-ncg-lstsq":
402            opt = minimize(
403                fun=loglik_func,
404                x0=np.linalg.lstsq(scaled_Z, Y, rcond=None)[0].flatten(
405                    order="F"
406                ),
407                jac=grad_func,
408                hess=hessian_func,
409                method="trust-ncg",
410            )
411            self.beta_ = opt.x
412            self.minloglik_ = opt.fun
413
414        self.classes_ = np.unique(y)
415
416        return self

Fit Ridge model to training data (X, y).

for beta: regression coeffs (beta11, ..., beta1p, ..., betaK1, ..., betaKp) for K classes and p covariates.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, **kwargs):
418    def predict(self, X, **kwargs):
419        """Predict test data X.
420
421        Args:
422
423            X: {array-like}, shape = [n_samples, n_features]
424                Training vectors, where n_samples is the number
425                of samples and n_features is the number of features.
426
427            **kwargs: additional parameters to be passed to
428                    self.cook_test_set
429
430        Returns:
431
432            model predictions: {array-like}
433        """
434
435        return np.argmax(self.predict_proba(X, **kwargs), axis=1)

Predict test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
def predict_proba(self, X, **kwargs):
437    def predict_proba(self, X, **kwargs):
438        """Predict probabilities for test data X.
439
440        Args:
441
442            X: {array-like}, shape = [n_samples, n_features]
443                Training vectors, where n_samples is the number
444                of samples and n_features is the number of features.
445
446            **kwargs: additional parameters to be passed to
447                    self.cook_test_set
448
449        Returns:
450
451            probability estimates for test data: {array-like}
452
453        """
454        if len(X.shape) == 1:
455            n_features = X.shape[0]
456            new_X = mo.rbind(
457                X.reshape(1, n_features),
458                np.ones(n_features).reshape(1, n_features),
459            )
460
461            Z = self.cook_test_set(new_X, **kwargs)
462
463        else:
464            Z = self.cook_test_set(X, **kwargs)
465
466        ZB = mo.safe_sparse_dot(
467            a=Z,
468            b=self.beta_.reshape(
469                self.n_classes,
470                X.shape[1] + self.n_hidden_features + self.n_clusters,
471            ).T,
472            backend=self.backend,
473        )
474
475        exp_ZB = np.exp(ZB)
476
477        return exp_ZB / exp_ZB.sum(axis=1)[:, None]

Predict probabilities for test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

probability estimates for test data: {array-like}
class Ridge2MultitaskClassifier(nnetsauce.ridge2.ridge2.Ridge2, sklearn.base.ClassifierMixin):
 21class Ridge2MultitaskClassifier(Ridge2, ClassifierMixin):
 22    """Multitask Ridge classification with 2 regularization parameters
 23
 24    Parameters:
 25
 26        n_hidden_features: int
 27            number of nodes in the hidden layer
 28
 29        activation_name: str
 30            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 31
 32        a: float
 33            hyperparameter for 'prelu' or 'elu' activation function
 34
 35        nodes_sim: str
 36            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 37            'uniform'
 38
 39        bias: boolean
 40            indicates if the hidden layer contains a bias term (True) or not
 41            (False)
 42
 43        dropout: float
 44            regularization parameter; (random) percentage of nodes dropped out
 45            of the training
 46
 47        n_clusters: int
 48            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 49                no clustering)
 50
 51        cluster_encode: bool
 52            defines how the variable containing clusters is treated (default is one-hot)
 53            if `False`, then labels are used, without one-hot encoding
 54
 55        type_clust: str
 56            type of clustering method: currently k-means ('kmeans') or Gaussian
 57            Mixture Model ('gmm')
 58
 59        type_scaling: a tuple of 3 strings
 60            scaling methods for inputs, hidden layer, and clustering respectively
 61            (and when relevant).
 62            Currently available: standardization ('std') or MinMax scaling ('minmax')
 63
 64        lambda1: float
 65            regularization parameter on direct link
 66
 67        lambda2: float
 68            regularization parameter on hidden layer
 69
 70        seed: int
 71            reproducibility seed for nodes_sim=='uniform'
 72
 73        backend: str
 74            "cpu" or "gpu" or "tpu"
 75
 76    Attributes:
 77
 78        beta_: {array-like}
 79            regression coefficients
 80
 81    Examples:
 82
 83    See also [https://github.com/Techtonique/nnetsauce/blob/master/examples/ridgemtask_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/ridgemtask_classification.py)
 84
 85    ```python
 86    import nnetsauce as ns
 87    import numpy as np
 88    from sklearn.datasets import load_breast_cancer
 89    from sklearn.model_selection import train_test_split
 90    from sklearn import metrics
 91    from time import time
 92
 93    breast_cancer = load_breast_cancer()
 94    Z = breast_cancer.data
 95    t = breast_cancer.target
 96    np.random.seed(123)
 97    X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2)
 98
 99    fit_obj = ns.Ridge2MultitaskClassifier(n_hidden_features=int(9.83730469e+01),
100                                    dropout=4.31054687e-01,
101                                    n_clusters=int(1.71484375e+00),
102                                    lambda1=1.24023438e+01, lambda2=7.30263672e+03)
103
104    start = time()
105    fit_obj.fit(X_train, y_train)
106    print(f"Elapsed {time() - start}")
107
108    print(fit_obj.score(X_test, y_test))
109    print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
110
111    start = time()
112    preds = fit_obj.predict(X_test)
113    print(f"Elapsed {time() - start}")
114    print(metrics.classification_report(preds, y_test))
115    ```
116
117    """
118
119    # construct the object -----
120
121    def __init__(
122        self,
123        n_hidden_features=5,
124        activation_name="relu",
125        a=0.01,
126        nodes_sim="sobol",
127        bias=True,
128        dropout=0,
129        n_clusters=2,
130        cluster_encode=True,
131        type_clust="kmeans",
132        type_scaling=("std", "std", "std"),
133        lambda1=0.1,
134        lambda2=0.1,
135        seed=123,
136        backend="cpu",
137    ):
138        super().__init__(
139            n_hidden_features=n_hidden_features,
140            activation_name=activation_name,
141            a=a,
142            nodes_sim=nodes_sim,
143            bias=bias,
144            dropout=dropout,
145            n_clusters=n_clusters,
146            cluster_encode=cluster_encode,
147            type_clust=type_clust,
148            type_scaling=type_scaling,
149            lambda1=lambda1,
150            lambda2=lambda2,
151            seed=seed,
152            backend=backend,
153        )
154
155        self.type_fit = "classification"
156
157    def fit(self, X, y, **kwargs):
158        """Fit Ridge model to training data (X, y).
159
160        Args:
161
162            X: {array-like}, shape = [n_samples, n_features]
163                Training vectors, where n_samples is the number
164                of samples and n_features is the number of features.
165
166            y: array-like, shape = [n_samples]
167                Target values.
168
169            **kwargs: additional parameters to be passed to
170                    self.cook_training_set or self.obj.fit
171
172        Returns:
173
174            self: object
175
176        """
177
178        sys_platform = platform.system()
179
180        assert mx.is_factor(y), "y must contain only integers"
181
182        self.classes_ = np.unique(y)  # for compatibility with sklearn
183        self.n_classes_ = len(self.classes_)  # for compatibility with sklearn
184
185        output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
186
187        n_X, p_X = X.shape
188        n_Z, p_Z = scaled_Z.shape
189
190        self.n_classes = len(np.unique(y))
191
192        # multitask response
193        Y = mo.one_hot_encode2(output_y, self.n_classes)
194
195        if self.n_clusters > 0:
196            if self.encode_clusters == True:
197                n_features = p_X + self.n_clusters
198            else:
199                n_features = p_X + 1
200        else:
201            n_features = p_X
202
203        X_ = scaled_Z[:, 0:n_features]
204        Phi_X_ = scaled_Z[:, n_features:p_Z]
205
206        B = mo.crossprod(x=X_, backend=self.backend) + self.lambda1 * np.diag(
207            np.repeat(1, X_.shape[1])
208        )
209        C = mo.crossprod(x=Phi_X_, y=X_, backend=self.backend)
210        D = mo.crossprod(
211            x=Phi_X_, backend=self.backend
212        ) + self.lambda2 * np.diag(np.repeat(1, Phi_X_.shape[1]))
213
214        if sys_platform in ("Linux", "Darwin"):
215            B_inv = pinv(B) if self.backend == "cpu" else jpinv(B)
216        else:
217            B_inv = pinv(B)
218
219        W = mo.safe_sparse_dot(a=C, b=B_inv, backend=self.backend)
220        S_mat = D - mo.tcrossprod(x=W, y=C, backend=self.backend)
221
222        if sys_platform in ("Linux", "Darwin"):
223            S_inv = pinv(S_mat) if self.backend == "cpu" else jpinv(S_mat)
224        else:
225            S_inv = pinv(S_mat)
226
227        Y2 = mo.safe_sparse_dot(a=S_inv, b=W, backend=self.backend)
228        inv = mo.rbind(
229            mo.cbind(
230                x=B_inv + mo.crossprod(x=W, y=Y2, backend=self.backend),
231                y=-np.transpose(Y2),
232                backend=self.backend,
233            ),
234            mo.cbind(x=-Y2, y=S_inv, backend=self.backend),
235            backend=self.backend,
236        )
237
238        self.beta_ = mo.safe_sparse_dot(
239            a=inv,
240            b=mo.crossprod(x=scaled_Z, y=Y, backend=self.backend),
241            backend=self.backend,
242        )
243        self.classes_ = np.unique(y)
244        return self
245
246    def predict(self, X, **kwargs):
247        """Predict test data X.
248
249        Args:
250
251            X: {array-like}, shape = [n_samples, n_features]
252                Training vectors, where n_samples is the number
253                of samples and n_features is the number of features.
254
255            **kwargs: additional parameters to be passed to
256                    self.cook_test_set
257
258        Returns:
259
260            model predictions: {array-like}
261
262        """
263
264        return np.argmax(self.predict_proba(X, **kwargs), axis=1)
265
266    def predict_proba(self, X, **kwargs):
267        """Predict probabilities for test data X.
268
269        Args:
270
271            X: {array-like}, shape = [n_samples, n_features]
272                Training vectors, where n_samples is the number
273                of samples and n_features is the number of features.
274
275            **kwargs: additional parameters to be passed to
276                    self.cook_test_set
277
278        Returns:
279
280            probability estimates for test data: {array-like}
281
282        """
283
284        if len(X.shape) == 1:
285            n_features = X.shape[0]
286            new_X = mo.rbind(
287                x=X.reshape(1, n_features),
288                y=np.ones(n_features).reshape(1, n_features),
289                backend=self.backend,
290            )
291
292            Z = self.cook_test_set(new_X, **kwargs)
293
294        else:
295            Z = self.cook_test_set(X, **kwargs)
296
297        ZB = mo.safe_sparse_dot(a=Z, b=self.beta_, backend=self.backend)
298
299        exp_ZB = np.exp(ZB)
300
301        return exp_ZB / exp_ZB.sum(axis=1)[:, None]

Multitask Ridge classification with 2 regularization parameters

Parameters:

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

lambda1: float
    regularization parameter on direct link

lambda2: float
    regularization parameter on hidden layer

seed: int
    reproducibility seed for nodes_sim=='uniform'

backend: str
    "cpu" or "gpu" or "tpu"

Attributes:

beta_: {array-like}
    regression coefficients

Examples:

See also https://github.com/Techtonique/nnetsauce/blob/master/examples/ridgemtask_classification.py

import nnetsauce as ns
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn import metrics
from time import time

breast_cancer = load_breast_cancer()
Z = breast_cancer.data
t = breast_cancer.target
np.random.seed(123)
X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2)

fit_obj = ns.Ridge2MultitaskClassifier(n_hidden_features=int(9.83730469e+01),
                                dropout=4.31054687e-01,
                                n_clusters=int(1.71484375e+00),
                                lambda1=1.24023438e+01, lambda2=7.30263672e+03)

start = time()
fit_obj.fit(X_train, y_train)
print(f"Elapsed {time() - start}")

print(fit_obj.score(X_test, y_test))
print(fit_obj.score(X_test, y_test, scoring="roc_auc"))

start = time()
preds = fit_obj.predict(X_test)
print(f"Elapsed {time() - start}")
print(metrics.classification_report(preds, y_test))
def fit(self, X, y, **kwargs):
157    def fit(self, X, y, **kwargs):
158        """Fit Ridge model to training data (X, y).
159
160        Args:
161
162            X: {array-like}, shape = [n_samples, n_features]
163                Training vectors, where n_samples is the number
164                of samples and n_features is the number of features.
165
166            y: array-like, shape = [n_samples]
167                Target values.
168
169            **kwargs: additional parameters to be passed to
170                    self.cook_training_set or self.obj.fit
171
172        Returns:
173
174            self: object
175
176        """
177
178        sys_platform = platform.system()
179
180        assert mx.is_factor(y), "y must contain only integers"
181
182        self.classes_ = np.unique(y)  # for compatibility with sklearn
183        self.n_classes_ = len(self.classes_)  # for compatibility with sklearn
184
185        output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
186
187        n_X, p_X = X.shape
188        n_Z, p_Z = scaled_Z.shape
189
190        self.n_classes = len(np.unique(y))
191
192        # multitask response
193        Y = mo.one_hot_encode2(output_y, self.n_classes)
194
195        if self.n_clusters > 0:
196            if self.encode_clusters == True:
197                n_features = p_X + self.n_clusters
198            else:
199                n_features = p_X + 1
200        else:
201            n_features = p_X
202
203        X_ = scaled_Z[:, 0:n_features]
204        Phi_X_ = scaled_Z[:, n_features:p_Z]
205
206        B = mo.crossprod(x=X_, backend=self.backend) + self.lambda1 * np.diag(
207            np.repeat(1, X_.shape[1])
208        )
209        C = mo.crossprod(x=Phi_X_, y=X_, backend=self.backend)
210        D = mo.crossprod(
211            x=Phi_X_, backend=self.backend
212        ) + self.lambda2 * np.diag(np.repeat(1, Phi_X_.shape[1]))
213
214        if sys_platform in ("Linux", "Darwin"):
215            B_inv = pinv(B) if self.backend == "cpu" else jpinv(B)
216        else:
217            B_inv = pinv(B)
218
219        W = mo.safe_sparse_dot(a=C, b=B_inv, backend=self.backend)
220        S_mat = D - mo.tcrossprod(x=W, y=C, backend=self.backend)
221
222        if sys_platform in ("Linux", "Darwin"):
223            S_inv = pinv(S_mat) if self.backend == "cpu" else jpinv(S_mat)
224        else:
225            S_inv = pinv(S_mat)
226
227        Y2 = mo.safe_sparse_dot(a=S_inv, b=W, backend=self.backend)
228        inv = mo.rbind(
229            mo.cbind(
230                x=B_inv + mo.crossprod(x=W, y=Y2, backend=self.backend),
231                y=-np.transpose(Y2),
232                backend=self.backend,
233            ),
234            mo.cbind(x=-Y2, y=S_inv, backend=self.backend),
235            backend=self.backend,
236        )
237
238        self.beta_ = mo.safe_sparse_dot(
239            a=inv,
240            b=mo.crossprod(x=scaled_Z, y=Y, backend=self.backend),
241            backend=self.backend,
242        )
243        self.classes_ = np.unique(y)
244        return self

Fit Ridge model to training data (X, y).

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, **kwargs):
246    def predict(self, X, **kwargs):
247        """Predict test data X.
248
249        Args:
250
251            X: {array-like}, shape = [n_samples, n_features]
252                Training vectors, where n_samples is the number
253                of samples and n_features is the number of features.
254
255            **kwargs: additional parameters to be passed to
256                    self.cook_test_set
257
258        Returns:
259
260            model predictions: {array-like}
261
262        """
263
264        return np.argmax(self.predict_proba(X, **kwargs), axis=1)

Predict test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
def predict_proba(self, X, **kwargs):
266    def predict_proba(self, X, **kwargs):
267        """Predict probabilities for test data X.
268
269        Args:
270
271            X: {array-like}, shape = [n_samples, n_features]
272                Training vectors, where n_samples is the number
273                of samples and n_features is the number of features.
274
275            **kwargs: additional parameters to be passed to
276                    self.cook_test_set
277
278        Returns:
279
280            probability estimates for test data: {array-like}
281
282        """
283
284        if len(X.shape) == 1:
285            n_features = X.shape[0]
286            new_X = mo.rbind(
287                x=X.reshape(1, n_features),
288                y=np.ones(n_features).reshape(1, n_features),
289                backend=self.backend,
290            )
291
292            Z = self.cook_test_set(new_X, **kwargs)
293
294        else:
295            Z = self.cook_test_set(X, **kwargs)
296
297        ZB = mo.safe_sparse_dot(a=Z, b=self.beta_, backend=self.backend)
298
299        exp_ZB = np.exp(ZB)
300
301        return exp_ZB / exp_ZB.sum(axis=1)[:, None]

Predict probabilities for test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

probability estimates for test data: {array-like}
class SubSampler:
 6class SubSampler:
 7    """Subsampling class.
 8
 9    Attributes:
10
11       y: array-like, shape = [n_samples]
12           Target values.
13
14       row_sample: double
15           subsampling fraction
16
17       n_samples: int
18            subsampling by using the number of rows (supersedes row_sample)
19
20       seed: int
21           reproductibility seed
22
23       n_jobs: int
24            number of jobs to run in parallel
25
26       verbose: bool
27            print progress messages and bars
28    """
29
30    def __init__(
31        self,
32        y,
33        row_sample=0.8,
34        n_samples=None,
35        seed=123,
36        n_jobs=None,
37        verbose=False,
38    ):
39        self.y = y
40        self.n_samples = n_samples
41        if self.n_samples is None:
42            assert (
43                row_sample < 1 and row_sample >= 0
44            ), "'row_sample' must be provided, plus < 1 and >= 0"
45            self.row_sample = row_sample
46        else:
47            assert self.n_samples < len(y), "'n_samples' must be < len(y)"
48            self.row_sample = self.n_samples / len(y)
49        self.seed = seed
50        self.indices = None
51        self.n_jobs = n_jobs
52        self.verbose = verbose
53
54    def subsample(self):
55        """Returns indices of subsampled input data.
56
57        Examples:
58
59        <ul>
60            <li> <a href="https://github.com/Techtonique/nnetsauce/blob/master/nnetsauce/demo/thierrymoudiki_20240105_subsampling.ipynb">20240105_subsampling.ipynb</a> </li>
61            <li> <a href="https://github.com/Techtonique/nnetsauce/blob/master/nnetsauce/demo/thierrymoudiki_20240131_subsampling_nsamples.ipynb">20240131_subsampling_nsamples.ipynb</a> </li>
62        </ul>
63
64        """
65        self.indices = dosubsample(
66            y=self.y,
67            row_sample=self.row_sample,
68            seed=self.seed,
69            n_jobs=self.n_jobs,
70            verbose=self.verbose,
71        )
72        return self.indices

Subsampling class.

Attributes:

y: array-like, shape = [n_samples] Target values.

row_sample: double subsampling fraction

n_samples: int subsampling by using the number of rows (supersedes row_sample)

seed: int reproductibility seed

n_jobs: int number of jobs to run in parallel

verbose: bool print progress messages and bars

def subsample(self):
54    def subsample(self):
55        """Returns indices of subsampled input data.
56
57        Examples:
58
59        <ul>
60            <li> <a href="https://github.com/Techtonique/nnetsauce/blob/master/nnetsauce/demo/thierrymoudiki_20240105_subsampling.ipynb">20240105_subsampling.ipynb</a> </li>
61            <li> <a href="https://github.com/Techtonique/nnetsauce/blob/master/nnetsauce/demo/thierrymoudiki_20240131_subsampling_nsamples.ipynb">20240131_subsampling_nsamples.ipynb</a> </li>
62        </ul>
63
64        """
65        self.indices = dosubsample(
66            y=self.y,
67            row_sample=self.row_sample,
68            seed=self.seed,
69            n_jobs=self.n_jobs,
70            verbose=self.verbose,
71        )
72        return self.indices

Returns indices of subsampled input data.

Examples: